Support thread local storage

examples/crashreport.c

@@ -8,6 +8,7 @@
 ╚─────────────────────────────────────────────────────────────────*/
 #endif
 #include "libc/log/log.h"
+#include "libc/runtime/symbols.internal.h"
 
 /**
  * @fileoverview How to print backtraces and cpu state on crash.

examples/examples.mk

@@ -143,14 +143,6 @@ o/$(MODE)/examples/nesemu1.com:							\
 	@$(COMPILE) -AZIP -T$@ o/$(MODE)/third_party/zip/zip.com -0qj $@	\
 		o/$(MODE)/examples/.nesemu1/.symtab
 
-o/$(MODE)/examples/hello.com.dbg:						\
-		$(EXAMPLES_DEPS)						\
-		o/$(MODE)/examples/hello.o					\
-		o/$(MODE)/examples/examples.pkg					\
-		$(CRT)								\
-		$(APE_NO_MODIFY_SELF)
-	@$(APELINK)
-
 o/$(MODE)/examples/nesemu1.o: QUOTA += -M512m
 
 $(EXAMPLES_OBJS): examples/examples.mk

examples/greenbean.c

@@ -16,6 +16,7 @@
 │ TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR             │
 │ PERFORMANCE OF THIS SOFTWARE.                                                │
 ╚─────────────────────────────────────────────────────────────────────────────*/
+#include "libc/assert.h"
 #include "libc/bits/atomic.h"
 #include "libc/calls/calls.h"
 #include "libc/calls/sigbits.h"
@@ -27,6 +28,7 @@
 #include "libc/intrin/kprintf.h"
 #include "libc/log/check.h"
 #include "libc/log/log.h"
+#include "libc/macros.internal.h"
 #include "libc/mem/mem.h"
 #include "libc/runtime/runtime.h"
 #include "libc/sock/goodsocket.internal.h"
@@ -90,7 +92,8 @@
  *
  */
 
-#define THREADS   512
+#define PORT      8080
+#define THREADS   10000
 #define HEARTBEAT 100
 #define KEEPALIVE 5000
 #define LOGGING   0
@@ -100,120 +103,39 @@
   "Referrer-Policy: origin\r\n"   \
   "Cache-Control: private; max-age=0\r\n"
 
-////////////////////////////////////////////////////////////////////////////////
+int workers;
-// BEGIN: Chris Wellons's Public Domain GNU Atomics Library
+int messages;
-
+int connections;
-#define BARRIER_INC(x)     __atomic_add_fetch(x, 1, __ATOMIC_SEQ_CST)
-#define BARRIER_GET(x)     __atomic_load_n(x, __ATOMIC_SEQ_CST)
-#define ATOMIC_LOAD(q)     __atomic_load_n(q, __ATOMIC_ACQUIRE)
-#define ATOMIC_RLOAD(q)    __atomic_load_n(q, __ATOMIC_RELAXED)
-#define ATOMIC_STORE(q, v) __atomic_store_n(q, v, __ATOMIC_RELEASE)
-#define ATOMIC_ADD(q, c)   __atomic_add_fetch(q, c, __ATOMIC_RELEASE)
-#define ATOMIC_AND(q, m)   __atomic_and_fetch(q, m, __ATOMIC_RELEASE)
-#define ATOMIC_CAS(q, e, d) \
-  __atomic_compare_exchange_n(q, e, d, 0, __ATOMIC_RELEASE, __ATOMIC_RELAXED)
-
-// Return the array index for then next value to be pushed. The size of this
-// array must be (1 << exp) elements. Write the value into this array index,
-// then commit it. With a single-consumer queue, this element store need not
-// be atomic. The value will appear in the queue after the commit. Returns
-// -1 if the queue is full.
-static int queue_push(uint32_t *q, int exp) {
-  uint32_t r = ATOMIC_LOAD(q);
-  int mask = (1u << exp) - 1;
-  int head = r & mask;
-  int tail = r >> 16 & mask;
-  int next = (head + 1u) & mask;
-  if (r & 0x8000) {  // avoid overflow on commit
-    ATOMIC_AND(q, ~0x8000);
-  }
-  return next == tail ? -1 : head;
-}
-
-// Commits and completes the push operation. Do this after storing into the
-// array. This operation cannot fail.
-static void queue_push_commit(uint32_t *q) {
-  ATOMIC_ADD(q, 1);
-}
-
-// Return the array index for the next value to be popped. The size of this
-// array must be (1 << exp) elements. Read from this array index, then
-// commit the pop. This element load need not be atomic. The value will be
-// removed from the queue after the commit. Returns -1 if the queue is
-// empty.
-static int queue_pop(uint32_t *q, int exp) {
-  uint32_t r = ATOMIC_LOAD(q);
-  int mask = (1u << exp) - 1;
-  int head = r & mask;
-  int tail = r >> 16 & mask;
-  return head == tail ? -1 : tail;
-}
-
-// Commits and completes the pop operation. Do this after loading from the
-// array. This operation cannot fail.
-static void queue_pop_commit(uint32_t *q) {
-  ATOMIC_ADD(q, 0x10000);
-}
-
-// Like queue_pop() but for multiple-consumer queues. The element load must
-// be atomic since it is concurrent with the producer's push, though it can
-// use a relaxed memory order. The loaded value must not be used unless the
-// commit is successful. Stores a temporary "save" to be used at commit.
-static int queue_mpop(uint32_t *q, int exp, uint32_t *save) {
-  uint32_t r = *save = ATOMIC_LOAD(q);
-  int mask = (1u << exp) - 1;
-  int head = r & mask;
-  int tail = r >> 16 & mask;
-  return head == tail ? -1 : tail;
-}
-
-// Like queue_pop_commit() but for multiple-consumer queues. It may fail if
-// another consumer pops concurrently, in which case the pop must be retried
-// from the beginning.
-static bool queue_mpop_commit(uint32_t *q, uint32_t save) {
-  return ATOMIC_CAS(q, &save, save + 0x10000);
-}
-
-// Spin-lock barrier for n threads, where n is a power of two.
-// Initialize *barrier to zero.
-static void barrier_waitn(int *barrier, int n) {
-  int v = BARRIER_INC(barrier);
-  if (v & (n - 1)) {
-    for (v &= n; (BARRIER_GET(barrier) & n) == v;) {
-      donothing;
-    }
-  }
-}
-
-// END: Chris Wellons's Public Domain GNU Atomics Library
-////////////////////////////////////////////////////////////////////////////////
-
-int barrier1;
-int itsbegun;
 int closingtime;
-int barrier2;
+const char *status;
-int itsdone;
 
 int Worker(void *id) {
   int server, yes = 1;
 
-  kprintf(" %d", id);
-  barrier_waitn(&barrier1, THREADS);
-  itsbegun = true;
-
   // load balance incoming connections for port 8080 across all threads
   // hangup on any browser clients that lag for more than a few seconds
   struct timeval timeo = {KEEPALIVE / 1000, KEEPALIVE % 1000};
-  struct sockaddr_in addr = {.sin_family = AF_INET, .sin_port = htons(8080)};
+  struct sockaddr_in addr = {.sin_family = AF_INET, .sin_port = htons(PORT)};
-  CHECK_NE(-1, (server = socket(AF_INET, SOCK_STREAM, IPPROTO_TCP)));
+
+  server = socket(AF_INET, SOCK_STREAM, IPPROTO_TCP);
+  if (server == -1) {
+    if (LOGGING) kprintf("%s() failed %m\n", "socket");
+    goto WorkerFinished;
+  }
+
   setsockopt(server, SOL_SOCKET, SO_RCVTIMEO, &timeo, sizeof(timeo));
   setsockopt(server, SOL_SOCKET, SO_SNDTIMEO, &timeo, sizeof(timeo));
   setsockopt(server, SOL_SOCKET, SO_REUSEADDR, &yes, sizeof(yes));
   setsockopt(server, SOL_SOCKET, SO_REUSEPORT, &yes, sizeof(yes));
   setsockopt(server, SOL_TCP, TCP_FASTOPEN, &yes, sizeof(yes));
   setsockopt(server, SOL_TCP, TCP_QUICKACK, &yes, sizeof(yes));
-  CHECK_EQ(0, bind(server, &addr, sizeof(addr)));
+
-  CHECK_EQ(0, listen(server, 10));
+  if (bind(server, &addr, sizeof(addr)) == -1) {
+    if (LOGGING) kprintf("%s() failed %m\n", "socket");
+    goto WorkerFinished;
+  }
+
+  listen(server, 10);
 
   // connection loop
   while (!closingtime) {
@@ -228,8 +150,12 @@ int Worker(void *id) {
     char inbuf[1500], outbuf[512], *p, *q;
     int clientip, client, inmsglen, outmsglen;
 
+    if (!IsLinux() &&
+        poll(&(struct pollfd){server, POLLIN}, 1, HEARTBEAT) < 1) {
+      continue;
+    }
+
     // wait for client connection
-    if (poll(&(struct pollfd){server, POLLIN}, 1, HEARTBEAT) < 1) continue;
     clientaddrsize = sizeof(clientaddr);
     client = accept(server, &clientaddr, &clientaddrsize);
 
@@ -245,6 +171,8 @@ int Worker(void *id) {
       continue;
     }
 
+    asm volatile("lock incl\t%0" : "+m"(connections));
+
     // message loop
     do {
       // parse the incoming http message
@@ -253,6 +181,7 @@ int Worker(void *id) {
       if ((got = read(client, inbuf, sizeof(inbuf))) <= 0) break;
       // check that client message wasn't fragmented into more reads
       if (!(inmsglen = ParseHttpMessage(&msg, inbuf, got))) break;
+      asm volatile("lock incl\t%0" : "+m"(messages));
 
 #if LOGGING
       // log the incoming http message
@@ -313,35 +242,46 @@ int Worker(void *id) {
              (msg.method == kHttpGet || msg.method == kHttpHead));
     DestroyHttpMessage(&msg);
     close(client);
+    asm volatile("lock decl\t%0" : "+m"(connections));
   }
 
   // inform the parent that this clone has finished
+WorkerFinished:
   close(server);
-  kprintf(" %d", id);
+  asm volatile("lock decl\t%0" : "+m"(workers));
-  barrier_waitn(&barrier2, THREADS);
-  itsdone = true;
   return 0;
 }
 
 void OnCtrlC(int sig) {
   closingtime = true;
+  status = " shutting down...";
 }
 
 int main(int argc, char *argv[]) {
-  /* ShowCrashReports(); */
+  int i;
-  int64_t loadtzdbearly;
+  uint32_t *hostips;
-  kprintf("welcome to greenbean\n");
+  ShowCrashReports();
-  gmtime(&loadtzdbearly);
+  sigaction(SIGINT, &(struct sigaction){.sa_handler = OnCtrlC}, 0);
-  for (int i = 0; i < THREADS; ++i) {
+  for (hostips = GetHostIps(), i = 0; hostips[i]; ++i) {
+    kprintf("listening on http://%d.%d.%d.%d:%d\n",
+            (hostips[i] & 0xff000000) >> 030, (hostips[i] & 0x00ff0000) >> 020,
+            (hostips[i] & 0x0000ff00) >> 010, (hostips[i] & 0x000000ff) >> 000,
+            PORT);
+  }
+  workers = THREADS;
+  for (i = 0; i < THREADS; ++i) {
     void *stack = mmap(0, 65536, PROT_READ | PROT_WRITE,
                        MAP_STACK | MAP_ANONYMOUS, -1, 0);
     clone(Worker, stack, 65536,
           CLONE_THREAD | CLONE_VM | CLONE_FS | CLONE_FILES | CLONE_SIGHAND,
           (void *)(intptr_t)i, 0, 0, 0, 0);
   }
-  while (!ATOMIC_LOAD(&itsbegun)) usleep(HEARTBEAT * 1000);
+  status = "";
-  sigaction(SIGINT, &(struct sigaction){.sa_handler = OnCtrlC}, 0);
+  while (workers) {
-  kprintf("\nit's begun\n");
+    kprintf(
-  while (!ATOMIC_LOAD(&itsdone)) usleep(HEARTBEAT * 1000);
+        "\r\e[K\e[32mgreenbean\e[0m workers=%d connections=%d messages=%d%s ",
-  kprintf("\nthank you for flying greenbean\n");
+        workers, connections, messages, status);
+    usleep(HEARTBEAT * 1000);
+  }
+  kprintf("\r\e[K");
 }

libc/calls/calls.mk

@@ -65,10 +65,21 @@ $(LIBC_CALLS_A).pkg:					\
 		$(LIBC_CALLS_A_OBJS)			\
 		$(foreach x,$(LIBC_CALLS_A_DIRECTDEPS),$($(x)_A).pkg)
 
+o/$(MODE)/libc/calls/sigenter-freebsd.o			\
+o/$(MODE)/libc/calls/sigenter-netbsd.o			\
+o/$(MODE)/libc/calls/sigenter-openbsd.o			\
+o/$(MODE)/libc/calls/sigenter-xnu.o:			\
+		OVERRIDE_COPTS +=			\
+			-mno-fentry			\
+			-fno-stack-protector		\
+			-fno-sanitize=all
+
+o/$(MODE)/libc/calls/sys_mprotect.greg.o		\
 o/$(MODE)/libc/calls/vdsofunc.greg.o			\
 o/$(MODE)/libc/calls/directmap.o			\
 o/$(MODE)/libc/calls/directmap-nt.o			\
 o/$(MODE)/libc/calls/mapstack.greg.o			\
+o/$(MODE)/libc/calls/execve-nt.greg.o			\
 o/$(MODE)/libc/calls/getcwd.greg.o			\
 o/$(MODE)/libc/calls/getcwd-xnu.greg.o			\
 o/$(MODE)/libc/calls/getprogramexecutablename.greg.o	\
@@ -114,7 +125,7 @@ o/$(MODE)/libc/calls/execl.o				\
 o/$(MODE)/libc/calls/execle.o				\
 o/$(MODE)/libc/calls/execlp.o				\
 o/$(MODE)/libc/calls/copyfile.o				\
-o/$(MODE)/libc/calls/execve-nt.o			\
+o/$(MODE)/libc/calls/execve-nt.greg.o			\
 o/$(MODE)/libc/calls/linkat-nt.o			\
 o/$(MODE)/libc/calls/renameat-nt.o			\
 o/$(MODE)/libc/calls/execve-sysv.o			\

libc/calls/execve-nt.greg.c

@@ -16,44 +16,95 @@
 │ TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR             │
 │ PERFORMANCE OF THIS SOFTWARE.                                                │
 ╚─────────────────────────────────────────────────────────────────────────────*/
+#define ShouldUseMsabiAttribute() 1
+#include "libc/bits/weaken.h"
 #include "libc/calls/calls.h"
 #include "libc/calls/internal.h"
 #include "libc/calls/ntspawn.h"
+#include "libc/calls/strace.internal.h"
 #include "libc/mem/alloca.h"
 #include "libc/nt/accounting.h"
+#include "libc/nt/console.h"
 #include "libc/nt/enum/startf.h"
 #include "libc/nt/enum/status.h"
+#include "libc/nt/memory.h"
 #include "libc/nt/runtime.h"
 #include "libc/nt/struct/processinformation.h"
 #include "libc/nt/struct/startupinfo.h"
 #include "libc/nt/synchronization.h"
+#include "libc/nt/thunk/msabi.h"
+#include "libc/runtime/memtrack.internal.h"
 #include "libc/runtime/runtime.h"
 #include "libc/str/str.h"
+#include "libc/sysv/consts/at.h"
+#include "libc/sysv/consts/map.h"
 #include "libc/sysv/consts/o.h"
+#include "libc/sysv/consts/ok.h"
+#include "libc/sysv/errfuns.h"
+
+__msabi extern typeof(CloseHandle) *const __imp_CloseHandle;
+__msabi extern typeof(WaitForSingleObject) *const __imp_WaitForSingleObject;
+__msabi extern typeof(GetExitCodeProcess) *const __imp_GetExitCodeProcess;
+__msabi extern typeof(UnmapViewOfFile) *const __imp_UnmapViewOfFile;
+
+static noinstrument __msabi bool32
+BlockExecveConsoleEvent(uint32_t dwCtrlType) {
+  // block SIGINT and SIGQUIT in execve() parent process
+  return true;
+}
 
 textwindows int sys_execve_nt(const char *program, char *const argv[],
                               char *const envp[]) {
   int rc;
   size_t i;
   uint32_t dwExitCode;
+  struct MemoryIntervals *mm;
   struct NtStartupInfo startinfo;
   struct NtProcessInformation procinfo;
+
+  // this is a non-recoverable operation, so do some manual validation
+  if (sys_faccessat_nt(AT_FDCWD, program, X_OK, 0) == -1) {
+    return eacces();
+  }
+
+  //////////////////////////////////////////////////////////////////////////////
+  // execve operation is unrecoverable from this point
+
+  // close cloexec handles
+  for (i = 3; i < g_fds.n; ++i) {
+    if (g_fds.p[i].kind != kFdEmpty && (g_fds.p[i].flags & O_CLOEXEC)) {
+      __imp_CloseHandle(g_fds.p[i].handle);
+    }
+  }
+
   bzero(&startinfo, sizeof(startinfo));
   startinfo.cb = sizeof(struct NtStartupInfo);
   startinfo.dwFlags = kNtStartfUsestdhandles;
   startinfo.hStdInput = __getfdhandleactual(0);
   startinfo.hStdOutput = __getfdhandleactual(1);
   startinfo.hStdError = __getfdhandleactual(2);
+
+  // spawn the process
   rc = ntspawn(program, argv, envp, 0, 0, 0, 1, 0, 0, &startinfo, &procinfo);
-  if (rc == -1) return -1;
+  if (rc == -1) {
-  CloseHandle(g_fds.p[0].handle);
+    STRACE("panic: unrecoverable ntspawn(%#s) error: %m", program);
-  CloseHandle(g_fds.p[1].handle);
+    __imp_ExitProcess(6543);
-  CloseHandle(procinfo.hThread);
+  }
+
+  //////////////////////////////////////////////////////////////////////////////
+  // zombie shell process remains, to wait for child and propagate its exit
+  // code
+
+  __imp_CloseHandle(g_fds.p[0].handle);
+  __imp_CloseHandle(g_fds.p[1].handle);
+  __imp_CloseHandle(procinfo.hThread);
+  __imp_SetConsoleCtrlHandler((void *)BlockExecveConsoleEvent, 1);
   do {
-    WaitForSingleObject(procinfo.hProcess, -1);
+    __imp_WaitForSingleObject(procinfo.hProcess, -1);
     dwExitCode = kNtStillActive;
-    GetExitCodeProcess(procinfo.hProcess, &dwExitCode);
+    __imp_GetExitCodeProcess(procinfo.hProcess, &dwExitCode);
   } while (dwExitCode == kNtStillActive);
-  CloseHandle(procinfo.hProcess);
+  __imp_CloseHandle(procinfo.hProcess);
-  _Exit(dwExitCode);
+  __imp_ExitProcess(dwExitCode);
+  unreachable;
 }

libc/calls/execve.c

@@ -63,11 +63,6 @@ int execve(const char *prog, char *const argv[], char *const envp[]) {
       kprintf("})\n");
     }
 #endif
-    for (i = 3; i < g_fds.n; ++i) {
-      if (g_fds.p[i].kind != kFdEmpty && (g_fds.p[i].flags & O_CLOEXEC)) {
-        close(i);
-      }
-    }
     if (!IsWindows()) {
       rc = sys_execve(prog, argv, envp);
     } else {

libc/calls/internal.h

@@ -134,6 +134,7 @@ i32 __sys_pipe2(i32[hasatleast 2], u32) hidden;
 i32 __sys_sigprocmask(i32, const sigset *, sigset *, u64) hidden;
 i32 __sys_utimensat(i32, const char *, const struct timespec *, i32) hidden;
 i32 __sys_wait4(i32, i32 *, i32, struct rusage *) hidden;
+i32 sys_arch_prctl(i32, i64) hidden;
 i32 sys_chdir(const char *) hidden;
 i32 sys_chroot(const char *) hidden;
 i32 sys_clock_gettime(i32, struct timespec *) hidden;

libc/calls/ntcontext2linux.c

@@ -18,10 +18,11 @@
 ╚─────────────────────────────────────────────────────────────────────────────*/
 #include "libc/calls/internal.h"
 #include "libc/calls/ucontext.h"
+#include "libc/log/libfatal.internal.h"
 #include "libc/nt/struct/context.h"
 #include "libc/str/str.h"
 
-textwindows void _ntcontext2linux(ucontext_t *ctx, const struct NtContext *cr) {
+privileged void _ntcontext2linux(ucontext_t *ctx, const struct NtContext *cr) {
   if (!cr) return;
   ctx->uc_flags = cr->EFlags;
   ctx->uc_mcontext.eflags = cr->EFlags;
@@ -46,10 +47,10 @@ textwindows void _ntcontext2linux(ucontext_t *ctx, const struct NtContext *cr) {
   ctx->uc_mcontext.gs = cr->SegGs;
   ctx->uc_mcontext.fs = cr->SegFs;
   ctx->uc_mcontext.fpregs = &ctx->__fpustate;
-  memcpy(&ctx->__fpustate, &cr->FltSave, sizeof(ctx->__fpustate));
+  __repmovsb(&ctx->__fpustate, &cr->FltSave, sizeof(ctx->__fpustate));
 }
 
-textwindows void _ntlinux2context(struct NtContext *cr, const ucontext_t *ctx) {
+privileged void _ntlinux2context(struct NtContext *cr, const ucontext_t *ctx) {
   if (!cr) return;
   cr->EFlags = ctx->uc_flags;
   cr->EFlags = ctx->uc_mcontext.eflags;
@@ -73,5 +74,5 @@ textwindows void _ntlinux2context(struct NtContext *cr, const ucontext_t *ctx) {
   cr->SegCs = ctx->uc_mcontext.cs;
   cr->SegGs = ctx->uc_mcontext.gs;
   cr->SegFs = ctx->uc_mcontext.fs;
-  memcpy(&cr->FltSave, &ctx->__fpustate, sizeof(ctx->__fpustate));
+  __repmovsb(&cr->FltSave, &ctx->__fpustate, sizeof(ctx->__fpustate));
 }

libc/calls/sig2.c

@@ -22,6 +22,7 @@
 #include "libc/calls/strace.internal.h"
 #include "libc/intrin/cmpxchg.h"
 #include "libc/intrin/spinlock.h"
+#include "libc/log/libfatal.internal.h"
 #include "libc/macros.internal.h"
 #include "libc/runtime/internal.h"
 #include "libc/runtime/runtime.h"
@@ -90,8 +91,8 @@ static textwindows struct Signal *__sig_remove(void) {
  * @note called from main thread
  * @return true if EINTR should be returned by caller
  */
-static textwindows bool __sig_deliver(bool restartable, int sig, int si_code,
+static privileged bool __sig_deliver(bool restartable, int sig, int si_code,
-                                      ucontext_t *ctx) {
+                                     ucontext_t *ctx) {
   unsigned rva, flags;
   siginfo_t info, *infop;
   STRACE("delivering %G", sig);
@@ -113,7 +114,7 @@ static textwindows bool __sig_deliver(bool restartable, int sig, int si_code,
   // setup the somewhat expensive information args
   // only if they're requested by the user in sigaction()
   if (flags & SA_SIGINFO) {
-    bzero(&info, sizeof(info));
+    __repstosb(&info, 0, sizeof(info));
     info.si_signo = sig;
     info.si_code = si_code;
     infop = &info;
@@ -162,8 +163,8 @@ static textwindows bool __sig_isfatal(int sig) {
  * @param restartable can be used to suppress true return if SA_RESTART
  * @return true if signal was delivered
  */
-textwindows bool __sig_handle(bool restartable, int sig, int si_code,
+privileged bool __sig_handle(bool restartable, int sig, int si_code,
-                              ucontext_t *ctx) {
+                             ucontext_t *ctx) {
   bool delivered;
   switch (__sighandrvas[sig]) {
     case (intptr_t)SIG_DFL:

libc/calls/sigenter-freebsd.c

@@ -25,13 +25,13 @@
 #include "libc/calls/typedef/sigaction_f.h"
 #include "libc/calls/ucontext.h"
 #include "libc/intrin/kprintf.h"
-#include "libc/intrin/repstosb.h"
+#include "libc/log/libfatal.internal.h"
 #include "libc/macros.internal.h"
 #include "libc/str/str.h"
 #include "libc/sysv/consts/sa.h"
 
-void __sigenter_freebsd(int sig, struct siginfo_freebsd *freebsdinfo,
+privileged void __sigenter_freebsd(int sig, struct siginfo_freebsd *freebsdinfo,
-                        struct ucontext_freebsd *ctx) {
+                                   struct ucontext_freebsd *ctx) {
   int rva, flags;
   struct Goodies {
     ucontext_t uc;
@@ -43,14 +43,14 @@ void __sigenter_freebsd(int sig, struct siginfo_freebsd *freebsdinfo,
     if (~flags & SA_SIGINFO) {
       ((sigaction_f)(_base + rva))(sig, 0, 0);
     } else {
-      repstosb(&g, 0, sizeof(g));
+      __repstosb(&g, 0, sizeof(g));
       g.uc.uc_mcontext.fpregs = &g.uc.__fpustate;
       g.uc.uc_stack.ss_sp = ctx->uc_stack.ss_sp;
       g.uc.uc_stack.ss_size = ctx->uc_stack.ss_size;
       g.uc.uc_stack.ss_flags = ctx->uc_stack.ss_flags;
       g.uc.uc_flags = ctx->uc_flags;
-      memcpy(&g.uc.uc_sigmask, &ctx->uc_sigmask,
+      __repmovsb(&g.uc.uc_sigmask, &ctx->uc_sigmask,
-             MIN(sizeof(g.uc.uc_sigmask), sizeof(ctx->uc_sigmask)));
+                 MIN(sizeof(g.uc.uc_sigmask), sizeof(ctx->uc_sigmask)));
       g.uc.uc_mcontext.r8 = ctx->uc_mcontext.mc_r8;
       g.uc.uc_mcontext.r9 = ctx->uc_mcontext.mc_r9;
       g.uc.uc_mcontext.r10 = ctx->uc_mcontext.mc_r10;
@@ -73,7 +73,7 @@ void __sigenter_freebsd(int sig, struct siginfo_freebsd *freebsdinfo,
       g.uc.uc_mcontext.gs = ctx->uc_mcontext.mc_gs;
       g.uc.uc_mcontext.err = ctx->uc_mcontext.mc_err;
       g.uc.uc_mcontext.trapno = ctx->uc_mcontext.mc_trapno;
-      memcpy(&g.uc.__fpustate, &ctx->uc_mcontext.mc_fpstate, 512);
+      __repmovsb(&g.uc.__fpustate, &ctx->uc_mcontext.mc_fpstate, 512);
       g.si.si_signo = freebsdinfo->si_signo;
       g.si.si_errno = freebsdinfo->si_errno;
       g.si.si_code = freebsdinfo->si_code;
@@ -89,8 +89,8 @@ void __sigenter_freebsd(int sig, struct siginfo_freebsd *freebsdinfo,
       ctx->uc_stack.ss_size = g.uc.uc_stack.ss_size;
       ctx->uc_stack.ss_flags = g.uc.uc_stack.ss_flags;
       ctx->uc_flags = g.uc.uc_flags;
-      memcpy(&ctx->uc_sigmask, &g.uc.uc_sigmask,
+      __repmovsb(&ctx->uc_sigmask, &g.uc.uc_sigmask,
-             MIN(sizeof(g.uc.uc_sigmask), sizeof(ctx->uc_sigmask)));
+                 MIN(sizeof(g.uc.uc_sigmask), sizeof(ctx->uc_sigmask)));
       ctx->uc_mcontext.mc_rdi = g.uc.uc_mcontext.rdi;
       ctx->uc_mcontext.mc_rsi = g.uc.uc_mcontext.rsi;
       ctx->uc_mcontext.mc_rdx = g.uc.uc_mcontext.rdx;
@@ -113,7 +113,7 @@ void __sigenter_freebsd(int sig, struct siginfo_freebsd *freebsdinfo,
       ctx->uc_mcontext.mc_err = g.uc.uc_mcontext.err;
       ctx->uc_mcontext.mc_rip = g.uc.uc_mcontext.rip;
       ctx->uc_mcontext.mc_rsp = g.uc.uc_mcontext.rsp;
-      memcpy(&ctx->uc_mcontext.mc_fpstate, &g.uc.__fpustate, 512);
+      __repmovsb(&ctx->uc_mcontext.mc_fpstate, &g.uc.__fpustate, 512);
     }
   }
   /*

libc/calls/sigenter-netbsd.c

@@ -24,12 +24,13 @@
 #include "libc/calls/struct/ucontext-netbsd.internal.h"
 #include "libc/calls/typedef/sigaction_f.h"
 #include "libc/calls/ucontext.h"
+#include "libc/log/libfatal.internal.h"
 #include "libc/macros.internal.h"
 #include "libc/str/str.h"
 #include "libc/sysv/consts/sa.h"
 
-void __sigenter_netbsd(int sig, struct siginfo_netbsd *si,
+privileged void __sigenter_netbsd(int sig, struct siginfo_netbsd *si,
-                       struct ucontext_netbsd *ctx) {
+                                  struct ucontext_netbsd *ctx) {
   int rva, flags;
   ucontext_t uc;
   struct siginfo si2;
@@ -39,8 +40,8 @@ void __sigenter_netbsd(int sig, struct siginfo_netbsd *si,
     if (~flags & SA_SIGINFO) {
       ((sigaction_f)(_base + rva))(sig, 0, 0);
     } else {
-      bzero(&uc, sizeof(uc));
+      __repstosb(&uc, 0, sizeof(uc));
-      bzero(&si2, sizeof(si2));
+      __repstosb(&si2, 0, sizeof(si2));
       si2.si_signo = si->si_signo;
       si2.si_code = si->si_code;
       si2.si_errno = si->si_errno;
@@ -52,8 +53,8 @@ void __sigenter_netbsd(int sig, struct siginfo_netbsd *si,
       uc.uc_stack.ss_sp = ctx->uc_stack.ss_sp;
       uc.uc_stack.ss_size = ctx->uc_stack.ss_size;
       uc.uc_stack.ss_flags = ctx->uc_stack.ss_flags;
-      memcpy(&uc.uc_sigmask, &ctx->uc_sigmask,
+      __repmovsb(&uc.uc_sigmask, &ctx->uc_sigmask,
-             MIN(sizeof(uc.uc_sigmask), sizeof(ctx->uc_sigmask)));
+                 MIN(sizeof(uc.uc_sigmask), sizeof(ctx->uc_sigmask)));
       uc.uc_mcontext.rdi = ctx->uc_mcontext.rdi;
       uc.uc_mcontext.rsi = ctx->uc_mcontext.rsi;
       uc.uc_mcontext.rdx = ctx->uc_mcontext.rdx;

libc/calls/sigenter-openbsd.c

@@ -24,13 +24,13 @@
 #include "libc/calls/struct/ucontext-openbsd.internal.h"
 #include "libc/calls/typedef/sigaction_f.h"
 #include "libc/calls/ucontext.h"
-#include "libc/intrin/repstosb.h"
+#include "libc/log/libfatal.internal.h"
 #include "libc/macros.internal.h"
 #include "libc/str/str.h"
 #include "libc/sysv/consts/sa.h"
 
-void __sigenter_openbsd(int sig, struct siginfo_openbsd *openbsdinfo,
+privileged void __sigenter_openbsd(int sig, struct siginfo_openbsd *openbsdinfo,
-                        struct ucontext_openbsd *ctx) {
+                                   struct ucontext_openbsd *ctx) {
   int rva, flags;
   struct Goodies {
     ucontext_t uc;
@@ -42,7 +42,7 @@ void __sigenter_openbsd(int sig, struct siginfo_openbsd *openbsdinfo,
     if (~flags & SA_SIGINFO) {
       ((sigaction_f)(_base + rva))(sig, 0, 0);
     } else {
-      repstosb(&g, 0, sizeof(g));
+      __repstosb(&g, 0, sizeof(g));
       g.si.si_signo = openbsdinfo->si_signo;
       g.si.si_code = openbsdinfo->si_code;
       g.si.si_errno = openbsdinfo->si_errno;
@@ -54,8 +54,8 @@ void __sigenter_openbsd(int sig, struct siginfo_openbsd *openbsdinfo,
       }
       g.si.si_value = openbsdinfo->si_value;
       g.uc.uc_mcontext.fpregs = &g.uc.__fpustate;
-      memcpy(&g.uc.uc_sigmask, &ctx->sc_mask,
+      __repmovsb(&g.uc.uc_sigmask, &ctx->sc_mask,
-             MIN(sizeof(g.uc.uc_sigmask), sizeof(ctx->sc_mask)));
+                 MIN(sizeof(g.uc.uc_sigmask), sizeof(ctx->sc_mask)));
       g.uc.uc_mcontext.rdi = ctx->sc_rdi;
       g.uc.uc_mcontext.rsi = ctx->sc_rsi;
       g.uc.uc_mcontext.rdx = ctx->sc_rdx;

libc/calls/sigenter-xnu.c

@@ -25,6 +25,7 @@
 #include "libc/calls/ucontext.h"
 #include "libc/intrin/kprintf.h"
 #include "libc/intrin/repstosb.h"
+#include "libc/log/libfatal.internal.h"
 #include "libc/str/str.h"
 #include "libc/sysv/consts/sa.h"
 
@@ -355,19 +356,19 @@ struct __darwin_ucontext {
   struct __darwin_mcontext64 *uc_mcontext;
 };
 
-noasan static void xnuexceptionstate2linux(
+static privileged void xnuexceptionstate2linux(
     mcontext_t *mc, struct __darwin_x86_exception_state64 *xnues) {
   mc->trapno = xnues->__trapno;
   mc->err = xnues->__err;
 }
 
-noasan static void linuxexceptionstate2xnu(
+static privileged void linuxexceptionstate2xnu(
     struct __darwin_x86_exception_state64 *xnues, mcontext_t *mc) {
   xnues->__trapno = mc->trapno;
   xnues->__err = mc->err;
 }
 
-noasan static void xnuthreadstate2linux(
+static privileged void xnuthreadstate2linux(
     mcontext_t *mc, struct __darwin_x86_thread_state64 *xnuss) {
   mc->rdi = xnuss->__rdi;
   mc->rsi = xnuss->__rsi;
@@ -392,7 +393,7 @@ noasan static void xnuthreadstate2linux(
   mc->r15 = xnuss->__r15;
 }
 
-noasan static void linuxthreadstate2xnu(
+static privileged void linuxthreadstate2xnu(
     struct __darwin_x86_thread_state64 *xnuss, ucontext_t *uc, mcontext_t *mc) {
   xnuss->__rdi = mc->rdi;
   xnuss->__rsi = mc->rsi;
@@ -417,14 +418,14 @@ noasan static void linuxthreadstate2xnu(
   xnuss->__r15 = mc->r15;
 }
 
-noasan static void CopyFpXmmRegs(void *d, const void *s) {
+static privileged void CopyFpXmmRegs(void *d, const void *s) {
   size_t i;
   for (i = 0; i < (8 + 16) * 16; i += 16) {
     __builtin_memcpy((char *)d + i, (const char *)s + i, 16);
   }
 }
 
-noasan static void xnussefpustate2linux(
+static privileged void xnussefpustate2linux(
     struct FpuState *fs, struct __darwin_x86_float_state64 *xnufs) {
   fs->cwd = xnufs->__fpu_fcw;
   fs->swd = xnufs->__fpu_fsw;
@@ -437,7 +438,7 @@ noasan static void xnussefpustate2linux(
   CopyFpXmmRegs(fs->st, &xnufs->__fpu_stmm0);
 }
 
-noasan static void linuxssefpustate2xnu(
+static privileged void linuxssefpustate2xnu(
     struct __darwin_x86_float_state64 *xnufs, struct FpuState *fs) {
   xnufs->__fpu_fcw = fs->cwd;
   xnufs->__fpu_fsw = fs->swd;
@@ -450,9 +451,9 @@ noasan static void linuxssefpustate2xnu(
   CopyFpXmmRegs(&xnufs->__fpu_stmm0, fs->st);
 }
 
-noasan void __sigenter_xnu(void *fn, int infostyle, int sig,
+privileged void __sigenter_xnu(void *fn, int infostyle, int sig,
-                           struct siginfo_xnu *xnuinfo,
+                               struct siginfo_xnu *xnuinfo,
-                           struct __darwin_ucontext *xnuctx) {
+                               struct __darwin_ucontext *xnuctx) {
   intptr_t ax;
   int rva, flags;
   struct Goodies {
@@ -465,7 +466,7 @@ noasan void __sigenter_xnu(void *fn, int infostyle, int sig,
     if (~flags & SA_SIGINFO) {
       ((sigaction_f)(_base + rva))(sig, 0, 0);
     } else {
-      repstosb(&g, 0, sizeof(g));
+      __repstosb(&g, 0, sizeof(g));
       if (xnuctx) {
         g.uc.uc_flags = xnuctx->uc_onstack ? SA_ONSTACK : 0;
         g.uc.uc_sigmask.__bits[0] = xnuctx->uc_sigmask;

libc/calls/wincrash.c

@@ -30,7 +30,7 @@
 #include "libc/sysv/consts/sicode.h"
 #include "libc/sysv/consts/sig.h"
 
-textwindows unsigned __wincrash(struct NtExceptionPointers *ep) {
+privileged unsigned __wincrash(struct NtExceptionPointers *ep) {
   int64_t rip;
   int sig, code;
   ucontext_t ctx;

libc/errno.h

@@ -8,7 +8,7 @@ COSMOPOLITAN_C_START_
  * @see libc/sysv/consts.sh for numbers
  */
 
-extern errno_t errno;
+#define errno (*__errno_location())
 
 /**
  * System call unavailable.
@@ -686,6 +686,10 @@ extern const long EXFULL;
 #define EXDEV           EXDEV
 #define EXFULL          EXFULL
 
+extern errno_t __errno;
+
+errno_t *__errno_location(void);
+
 COSMOPOLITAN_C_END_
 #endif /* !(__ASSEMBLER__ + __LINKER__ + 0) */
 #endif /* COSMOPOLITAN_LIBC_ERRNO_H_ */

libc/fmt/sprintf.c

@@ -16,8 +16,10 @@
 │ TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR             │
 │ PERFORMANCE OF THIS SOFTWARE.                                                │
 ╚─────────────────────────────────────────────────────────────────────────────*/
+#include "libc/bits/weaken.h"
 #include "libc/fmt/fmt.h"
 #include "libc/limits.h"
+#include "libc/log/log.h"
 
 /**
  * Formats string to buffer that's hopefully large enough.

libc/intrin/intrin.mk

@@ -66,6 +66,7 @@ o/$(MODE)/libc/intrin/kprintf.greg.o:			\
 
 o/$(MODE)/libc/intrin/tls.greg.o			\
 o/$(MODE)/libc/intrin/exit.greg.o			\
+o/$(MODE)/libc/intrin/errno.greg.o			\
 o/$(MODE)/libc/intrin/exit1.greg.o			\
 o/$(MODE)/libc/intrin/gettid.greg.o			\
 o/$(MODE)/libc/intrin/getenv.greg.o			\

libc/intrin/kprintf.greg.c

@@ -262,6 +262,7 @@ privileged static size_t kformat(char *b, size_t n, const char *fmt, va_list va,
           continue;
 
         case '#':
+        case '`':
           hash = '0';
           continue;
 

libc/intrin/spinlock.h

@@ -21,7 +21,7 @@
 #define _seizelock(lock)                        \
   do {                                          \
     typeof(*(lock)) x = 1;                      \
-    __atomic_store(lock, &x, __ATOMIC_SEQ_CST); \
+    __atomic_store(lock, &x, __ATOMIC_RELEASE); \
   } while (0)
 
 #define _spinlock_tiny(lock)  \

libc/intrin/threaded.internal.h

@@ -3,6 +3,7 @@
 #if !(__ASSEMBLER__ + __LINKER__ + 0)
 COSMOPOLITAN_C_START_
 
+extern bool __hastls;
 extern bool __threaded;
 
 COSMOPOLITAN_C_END_

libc/intrin/tls.greg.c

@@ -83,6 +83,9 @@ void *TlsGetValue(uint32_t dwTlsIndex) {
         : "=r"(lpTlsValue)
         : "m"(*((long *)0x1480 + dwTlsIndex)));
     return lpTlsValue;
+    // // this could also be written as...
+    // asm("movq\t%%gs:0x30,%0" : "=a"(tib));
+    // return (void *)tib[0x1480 / 8 + dwTlsIndex];
   } else {
     return __imp_TlsGetValue(dwTlsIndex);
   }

libc/log/backtrace2.greg.c

@@ -74,15 +74,18 @@ static int PrintBacktraceUsingAddr2line(int fd, const struct StackFrame *bp) {
     return -1;
   }
 
-  /*
+  // DWARF is a weak standard. Platforms that use LLVM or old GNU
-   * DWARF is a weak standard. Platforms that use LLVM or old GNU
+  // usually can't be counted upon to print backtraces correctly.
-   * usually can't be counted upon to print backtraces correctly.
-   */
   if (!IsLinux() && !IsWindows()) {
     ShowHint("won't print addr2line backtrace because probably llvm");
     return -1;
   }
 
+  if (IsWindows()) {
+    // TODO: We need a way to *not* pass //?/C:/... paths to mingw
+    return -1;
+  }
+
   i = 0;
   j = 0;
   argv[i++] = "addr2line";

libc/log/oncrash.c

@@ -277,6 +277,7 @@ relegated noinstrument void __oncrash(int sig, struct siginfo *si,
   intptr_t rip;
   int gdbpid, err;
   static bool noreentry, notpossible;
+  STRACE("__oncrash rip %x", ctx->uc_mcontext.rip);
   __atomic_fetch_sub(&g_ftrace, 1, __ATOMIC_RELAXED);
   __atomic_fetch_sub(&__strace, 1, __ATOMIC_RELAXED);
   if (_lockcmpxchg(&noreentry, false, true)) {

libc/macros.internal.inc

@@ -124,6 +124,15 @@
 	.section .privileged,"ax",@progbits
 .endm
 
+//	Loads address of errno into %rcx
+.macro	.errno
+	call	__errno_location
+//	cs
+//	cs
+//	cs
+//	mov	$__errno,%eax
+.endm
+
 //	Post-Initialization Read-Only (PIRO) BSS section.
 //	@param	ss is an optional string, for control image locality
 .macro	.piro	ss

libc/nexgen32e/hastls.c

@@ -0,0 +1,21 @@
+/*-*- mode:c;indent-tabs-mode:nil;c-basic-offset:2;tab-width:8;coding:utf-8 -*-│
+│vi: set net ft=c ts=2 sts=2 sw=2 fenc=utf-8                                :vi│
+╞══════════════════════════════════════════════════════════════════════════════╡
+│ Copyright 2022 Justine Alexandra Roberts Tunney                              │
+│                                                                              │
+│ Permission to use, copy, modify, and/or distribute this software for         │
+│ any purpose with or without fee is hereby granted, provided that the         │
+│ above copyright notice and this permission notice appear in all copies.      │
+│                                                                              │
+│ THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL                │
+│ WARRANTIES WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED                │
+│ WARRANTIES OF MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE             │
+│ AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL         │
+│ DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR        │
+│ PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER               │
+│ TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR             │
+│ PERFORMANCE OF THIS SOFTWARE.                                                │
+╚─────────────────────────────────────────────────────────────────────────────*/
+#include "libc/intrin/threaded.internal.h"
+
+bool __hastls;

libc/nexgen32e/nexgen32e.mk

@@ -42,7 +42,8 @@ $(LIBC_NEXGEN32E_A).pkg:				\
 		$(LIBC_NEXGEN32E_A_OBJS)		\
 		$(foreach x,$(LIBC_NEXGEN32E_A_DIRECTDEPS),$($(x)_A).pkg)
 
-o/$(MODE)/libc/nexgen32e/errno.o:			\
+o/$(MODE)/libc/nexgen32e/hastls.o			\
+o/$(MODE)/libc/nexgen32e/threaded.o:			\
 		OVERRIDE_CFLAGS +=			\
 			$(NO_MAGIC)			\
 			-fno-sanitize=all

libc/runtime/arch_prctl.c

@@ -19,6 +19,7 @@
 #include "libc/bits/asmflag.h"
 #include "libc/bits/bits.h"
 #include "libc/calls/calls.h"
+#include "libc/calls/internal.h"
 #include "libc/dce.h"
 #include "libc/nexgen32e/msr.h"
 #include "libc/nexgen32e/x86feature.h"
@@ -57,8 +58,6 @@
                    "d"((uint32_t)(val_ >> 32)));  \
   } while (0)
 
-int sys_arch_prctl(int, int64_t) hidden;
-
 static inline int arch_prctl_fsgsbase(int code, int64_t addr) {
   switch (code) {
     case ARCH_SET_GS:

libc/runtime/clone.greg.c

@@ -21,6 +21,7 @@
 #include "libc/calls/internal.h"
 #include "libc/calls/strace.internal.h"
 #include "libc/calls/struct/ucontext-netbsd.internal.h"
+#include "libc/dce.h"
 #include "libc/errno.h"
 #include "libc/intrin/asan.internal.h"
 #include "libc/intrin/kprintf.h"
@@ -34,6 +35,7 @@
 #include "libc/runtime/runtime.h"
 #include "libc/sysv/consts/clone.h"
 #include "libc/sysv/consts/nr.h"
+#include "libc/sysv/consts/nrlinux.h"
 #include "libc/sysv/errfuns.h"
 #include "libc/thread/freebsd.internal.h"
 #include "libc/thread/xnu.internal.h"
@@ -47,10 +49,56 @@ STATIC_YOINK("gettid");  // for kprintf()
 #define __NR__lwp_setprivate              317
 #define __NR_bsdthread_create             0x02000168
 #define __NR_thread_fast_set_cthread_self 0x03000003
+#define __NR_sysarch                      0x000000a5
+#define __NR___set_tcb                    0x00000149
 #define PTHREAD_START_CUSTOM_XNU          0x01000000
 #define LWP_DETACHED                      0x00000040
 #define LWP_SUSPENDED                     0x00000080
 
+char __tls[512];
+int __errno_global;
+extern int __errno_index;
+
+privileged void __setup_tls(void) {
+  int ax, dx;
+  uint64_t magic;
+  unsigned char *p;
+  *(intptr_t *)__tls = (intptr_t)__tls;
+  *(intptr_t *)(__tls + 0x30) = (intptr_t)__tls;
+  *(int *)(__tls + 0x3c) = __errno;
+  if (IsWindows()) {
+    __errno_index = TlsAlloc();
+    TlsSetValue(__errno_index, (void *)(intptr_t)__errno);
+  } else if (IsLinux()) {
+    asm volatile("syscall"
+                 : "=a"(ax)
+                 : "0"(__NR_linux_arch_prctl), "D"(ARCH_SET_FS), "S"(__tls)
+                 : "rcx", "r11", "memory");
+  } else if (IsFreebsd()) {
+    asm volatile("syscall"
+                 : "=a"(ax)
+                 : "0"(__NR_sysarch), "D"(129), "S"(__tls)
+                 : "rcx", "r11", "memory", "cc");
+  } else if (IsXnu()) {
+    asm volatile("syscall"
+                 : "=a"(ax)
+                 : "0"(__NR_thread_fast_set_cthread_self),
+                   "D"((intptr_t)__tls - 0x30)
+                 : "rcx", "r11", "memory", "cc");
+  } else if (IsOpenbsd()) {
+    asm volatile("syscall"
+                 : "=a"(ax)
+                 : "0"(__NR___set_tcb), "D"(__tls)
+                 : "rcx", "r11", "memory", "cc");
+  } else if (IsNetbsd()) {
+    asm volatile("syscall"
+                 : "=a"(ax), "=d"(dx)
+                 : "0"(__NR__lwp_setprivate), "D"(__tls)
+                 : "rcx", "r11", "memory", "cc");
+  }
+  __hastls = true;
+}
+
 uint32_t WinThreadThunk(void *warg);
 asm(".section\t.text.windows,\"ax\",@progbits\n\t"
     ".local\tWinThreadThunk\n"
@@ -58,7 +106,8 @@ asm(".section\t.text.windows,\"ax\",@progbits\n\t"
     "xor\t%ebp,%ebp\n\t"
     "mov\t%rcx,%rdi\n\t"
     "mov\t%rcx,%rsp\n\t"
-    "jmp\tWinThreadMain\n\t"
+    "and\t$-16,%rsp\n\t"
+    "call\tWinThreadMain\n\t"
     ".size\tWinThreadThunk,.-WinThreadThunk\n\t"
     ".previous");
 __attribute__((__used__, __no_reorder__))
@@ -69,7 +118,6 @@ WinThreadMain(struct WinThread *wt) {
   if (wt->flags & CLONE_CHILD_SETTID) {
     *wt->ctid = wt->tid;
   }
-  // TlsSetValue(__winthread, wt);
   rc = wt->func(wt->arg);
   if (wt->flags & CLONE_CHILD_CLEARTID) {
     *wt->ctid = 0;
@@ -380,32 +428,25 @@ static int CloneLinux(int (*func)(void *), char *stk, size_t stksz, int flags,
                       void *arg, int *ptid, void *tls, size_t tlssz,
                       int *ctid) {
   int ax;
-  bool failed;
-  intptr_t *stack;
   register int *r8 asm("r8") = tls;
-  register int (*r9)(void *) asm("r9") = func;
   register int *r10 asm("r10") = ctid;
-  stack = (intptr_t *)(stk + stksz);
+  register void *r9 asm("r9") = func;
-  *--stack = (long)arg;  // push 1
+  intptr_t *stack = (intptr_t *)(stk + stksz);
-  asm volatile("syscall"
+  *--stack = (intptr_t)arg;
-               : "=a"(ax)
+  asm volatile("syscall\n\t"
-               : "0"(__NR_clone_linux), "D"(flags), "S"(stack), "d"(ptid),
+               "test\t%0,%0\n\t"
-                 "r"(r10), "r"(r8), "r"(r9)
+               "jnz\t1f\n\t"
-               : "rcx", "r11", "memory");
+               "xor\t%%ebp,%%ebp\n\t"
-  if (ax > -4096u) {
+               "pop\t%%rdi\n\t"   // arg
-    errno = -ax;
+               "call\t*%%r9\n\t"  // func
-    return -1;
-  }
-  if (ax) return ax;
-  asm volatile("xor\t%%ebp,%%ebp\n\t"
-               "pop\t%%rdi\n\t"  // pop 1
-               "call\t*%0\n\t"
                "xchg\t%%eax,%%edi\n\t"
-               "jmp\t_Exit1"
+               "jmp\t_Exit1\n1:"
-               : /* no outputs */
+               : "=a"(ax)
-               : "r"(r9)
+               : "0"(__NR_clone_linux), "D"(flags), "S"(stack), "r"(r10),
-               : "memory");
+                 "r"(r8), "r"(r9)
-  unreachable;
+               : "rcx", "r11", "memory");
+  if (ax > -4096u) errno = -ax, ax = -1;
+  return ax;
 }
 
 /**
@@ -433,8 +474,8 @@ static int CloneLinux(int (*func)(void *), char *stk, size_t stksz, int flags,
  *     if the fork() or vfork() equivalent flags are used it's highly
  *     recommended that this value be GetStackSize(), or else kprintf
  *     and other runtime services providing memory safety can't do as
- *     good and quick of a job; this value must be 4096-aligned, plus
+ *     good and quick of a job; this value must be 16-aligned plus it
- *     it must be at minimum 4096 bytes in size
+ *     must be at minimum 4096 bytes in size
  * @param flags usually has one of
  *     - `SIGCHLD` will delegate to fork()
  *     - `CLONE_VFORK|CLONE_VM|SIGCHLD` means vfork()
@@ -454,15 +495,17 @@ static int CloneLinux(int (*func)(void *), char *stk, size_t stksz, int flags,
  * @param tlssz is the size of tls in bytes
  * @param ctid lets the child receive its thread id;
  *     this parameter is ignored if `CLONE_CHILD_SETTID` is not set
- * @return tid on success and 0 to the child, otherwise -1 w/ errno
+ * @return tid on success and 0 to the child, or -1 w/ errno
  * @threadsafe
  */
 int clone(int (*func)(void *), void *stk, size_t stksz, int flags, void *arg,
           int *ptid, void *tls, size_t tlssz, int *ctid) {
   int rc;
 
-  // let kprintf() switch from pids to tids
   __threaded = true;
+  if (tls && !__hastls) {
+    __setup_tls();
+  }
 
   // verify memory is kosher
   if (IsAsan() &&
@@ -504,7 +547,7 @@ int clone(int (*func)(void *), void *stk, size_t stksz, int flags, void *arg,
 
   // we now assume we're creating a thread
   // these platforms can't do signals the way linux does
-  else if (!IsTiny() && ((stksz < PAGESIZE || (stksz & (PAGESIZE - 1))) ||
+  else if (!IsTiny() && ((stksz < PAGESIZE || (stksz & 15)) ||
                          (flags & ~(CLONE_SETTLS | CLONE_PARENT_SETTID |
                                     CLONE_CHILD_SETTID)) !=
                              (CLONE_THREAD | CLONE_VM | CLONE_FS | CLONE_FILES |
@@ -521,15 +564,13 @@ int clone(int (*func)(void *), void *stk, size_t stksz, int flags, void *arg,
   }
 
   // These platforms can't do segment registers like linux does
-  else if (flags & CLONE_SETTLS) {
+  else if (IsWindows()) {
-    rc = einval();
-  } else if (IsWindows()) {
     rc = CloneWindows(func, stk, stksz, flags, arg, ptid, tls, tlssz, ctid);
   } else {
     rc = enosys();
   }
 
-  STRACE("clone(%p, %p, %'zu, %#x, %p, %p, %p, %'zu, %p) → %d% m", func, stk,
+  STRACE("clone(%p, %p, %'zu, %#x, %p, %p, %p, %'zu, %p) → %d", func, stk,
          stksz, flags, arg, ptid, tls, tlssz, ctid, rc);
   return rc;
 }

libc/runtime/ftracer.c

@@ -89,8 +89,8 @@ privileged noinstrument noasan noubsan void ftracer(void) {
     frame = __builtin_frame_address(0);
     frame = frame->next;
     if (frame->addr != g_lastaddr) {
-      kprintf("+ %*s%t %d\r\n", GetNestingLevel(frame) * 2, "", frame->addr,
+      kprintf("%rFUN %5P %'18T %*s%t\r\n", GetNestingLevel(frame) * 2, "",
-              ClocksToNanos(stamp, g_laststamp));
+              frame->addr);
       g_laststamp = X86_HAVE(RDTSCP) ? rdtscp(0) : rdtsc();
       g_lastaddr = frame->addr;
     }

libc/runtime/getinstructionlengths.c

@@ -0,0 +1,59 @@
+/*-*- mode:c;indent-tabs-mode:nil;c-basic-offset:2;tab-width:8;coding:utf-8 -*-│
+│vi: set net ft=c ts=2 sts=2 sw=2 fenc=utf-8                                :vi│
+╞══════════════════════════════════════════════════════════════════════════════╡
+│ Copyright 2022 Justine Alexandra Roberts Tunney                              │
+│                                                                              │
+│ Permission to use, copy, modify, and/or distribute this software for         │
+│ any purpose with or without fee is hereby granted, provided that the         │
+│ above copyright notice and this permission notice appear in all copies.      │
+│                                                                              │
+│ THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL                │
+│ WARRANTIES WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED                │
+│ WARRANTIES OF MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE             │
+│ AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL         │
+│ DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR        │
+│ PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER               │
+│ TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR             │
+│ PERFORMANCE OF THIS SOFTWARE.                                                │
+╚─────────────────────────────────────────────────────────────────────────────*/
+#include "libc/calls/calls.h"
+#include "libc/macros.internal.h"
+#include "libc/runtime/runtime.h"
+#include "libc/sysv/consts/map.h"
+#include "libc/sysv/consts/prot.h"
+#include "third_party/xed/x86.h"
+
+/**
+ * Returns lengths of x86 ops in binary.
+ *
+ * The first decoded instruction is at `_ereal`. Lengths can be 1 to 15
+ * bytes. Each byte in the return value is in that range, and the array
+ * is NUL terminated. The returned memory is memoized, since this costs
+ * some time. For example, for a 10mb Python binary, it takes 20 millis
+ * so the basic idea is is you can use this output multiple times which
+ * is a faster way to iterate over the binary than calling Xed.
+ *
+ * @return nul-terminated length array on success, or null
+ */
+privileged unsigned char *GetInstructionLengths(void) {
+  static bool once;
+  int i, n, err, len, rem;
+  static unsigned char *res;
+  struct XedDecodedInst xedd;
+  unsigned char *p, *mem, *code;
+  if (!once) {
+    if ((mem = mmap(0, ROUNDUP(__privileged_addr - _ereal + 1, FRAMESIZE),
+                    PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS, -1,
+                    0)) != MAP_FAILED) {
+      for (p = mem, code = _ereal; code < __privileged_addr; code += len) {
+        rem = __privileged_addr - code;
+        xed_decoded_inst_zero_set_mode(&xedd, XED_MACHINE_MODE_LONG_64);
+        err = xed_instruction_length_decode(&xedd, code, rem);
+        *p++ = len = !err ? xedd.length : 1;
+      }
+      res = mem;
+    }
+    once = true;
+  }
+  return res;
+}

libc/runtime/hook.greg.c

@@ -23,6 +23,7 @@
 #include "libc/calls/struct/sigset.h"
 #include "libc/dce.h"
 #include "libc/errno.h"
+#include "libc/intrin/kprintf.h"
 #include "libc/log/libfatal.internal.h"
 #include "libc/runtime/runtime.h"
 #include "libc/runtime/symbols.internal.h"
@@ -56,82 +57,69 @@ privileged noinstrument noasan int __hook(void *ifunc,
   uint64_t code, mcode;
   sigset_t mask, oldmask;
   intptr_t kMcount = (intptr_t)&mcount;
-  intptr_t kProgramCodeStart = (intptr_t)&_ereal;
+  intptr_t kProgramCodeStart = (intptr_t)_ereal;
-  intptr_t kPrivilegedStart = (intptr_t)&__privileged_start;
+  intptr_t kPrivilegedStart = (intptr_t)__privileged_addr;
-  bool kIsBinaryAligned = !(kPrivilegedStart & (PAGESIZE - 1));
+  __morph_begin();
-  if (!IsWindows()) {
+  for (i = 0; i < symbols->count; ++i) {
-    sigfillset(&mask);
+    if (symbols->addr_base + symbols->symbols[i].x < kProgramCodeStart) {
-    sys_sigprocmask(SIG_BLOCK, &mask, &oldmask);
+      continue;
-  }
+    }
-  if ((rc = mprotect(
+    if (symbols->addr_base + symbols->symbols[i].y >= kPrivilegedStart) {
-           (void *)symbols->addr_base, kPrivilegedStart - symbols->addr_base,
+      break;
-           kIsBinaryAligned ? PROT_READ | PROT_WRITE
+    }
-                            : PROT_READ | PROT_WRITE | PROT_EXEC)) != -1) {
+    for (p = (char *)symbols->addr_base + symbols->symbols[i].x,
-    for (i = 0; i < symbols->count; ++i) {
+        pe = (char *)symbols->addr_base + symbols->symbols[i].y;
-      if (symbols->addr_base + symbols->symbols[i].x < kProgramCodeStart) {
+         p + 8 - 1 <= pe; ++p) {
-        continue;
+      code = ((uint64_t)(255 & p[7]) << 070 | (uint64_t)(255 & p[6]) << 060 |
-      }
+              (uint64_t)(255 & p[5]) << 050 | (uint64_t)(255 & p[4]) << 040 |
-      if (symbols->addr_base + symbols->symbols[i].y >= kPrivilegedStart) {
+              (uint64_t)(255 & p[3]) << 030 | (uint64_t)(255 & p[2]) << 020 |
+              (uint64_t)(255 & p[1]) << 010 | (uint64_t)(255 & p[0]) << 000);
+
+      /*
+       * Test for -mrecord-mcount (w/ -fpie or -fpic)
+       *
+       *   nopw 0x00(%rax,%rax,1)  ← morphed by package.com
+       *   call *mcount(%rip)      ← linked w/o -static
+       *   addr32 call mcount      ← relaxed w/ -static
+       *   addr32 call mcount      ← relaxed w/ -static
+       *
+       * Note that gcc refuses to insert the six byte nop.
+       */
+      if ((code & 0x0000FFFFFFFFFFFF) == 0x0000441F0F66 ||
+          (code & 0x0000FFFFFFFFFFFF) ==
+              ((((kMcount - ((intptr_t)&p[2] + 4)) << 16) | 0xE867) &
+               0x0000FFFFFFFFFFFF) ||
+          (code & 0x0000FFFFFFFFFFFF) ==
+              ((((kMcount - ((intptr_t)&p[2] + 4)) << 16) | 0xFF15) &
+               0x0000FFFFFFFFFFFF)) {
+        p[0] = 0x67;
+        p[1] = 0xE8;
+        addr = (intptr_t)ifunc - ((intptr_t)&p[2] + 4);
+        p[2] = (addr & 0x000000ff) >> 000;
+        p[3] = (addr & 0x0000ff00) >> 010;
+        p[4] = (addr & 0x00ff0000) >> 020;
+        p[5] = (addr & 0xff000000) >> 030;
         break;
       }
-      for (p = (char *)symbols->addr_base + symbols->symbols[i].x,
-          pe = (char *)symbols->addr_base + symbols->symbols[i].y;
-           p + 8 - 1 <= pe; ++p) {
-        code = ((uint64_t)(255 & p[7]) << 070 | (uint64_t)(255 & p[6]) << 060 |
-                (uint64_t)(255 & p[5]) << 050 | (uint64_t)(255 & p[4]) << 040 |
-                (uint64_t)(255 & p[3]) << 030 | (uint64_t)(255 & p[2]) << 020 |
-                (uint64_t)(255 & p[1]) << 010 | (uint64_t)(255 & p[0]) << 000);
 
-        /*
+      /*
-         * Test for -mrecord-mcount (w/ -fpie or -fpic)
+       * Test for -mnop-mcount (w/ -fno-pie)
-         *
+       */
-         *   nopw 0x00(%rax,%rax,1)  ← morphed by package.com
+      mcode = code & 0x000000FFFFFFFFFF;
-         *   call *mcount(%rip)      ← linked w/o -static
+      if ((mcode == 0x00441F0F /*   nopl 0x00(%eax,%eax,1) [canonical] */) ||
-         *   addr32 call mcount      ← relaxed w/ -static
+          (mcode == 0x00041F0F67 /* nopl (%eax,%eax,1)     [older gcc] */)) {
-         *   addr32 call mcount      ← relaxed w/ -static
+        if (p[-1] != 0x66 /*        nopw 0x0(%rax,%rax,1)  [donotwant] */) {
-         *
+          p[0] = 0xE8 /* call Jvds */;
-         * Note that gcc refuses to insert the six byte nop.
+          addr = (intptr_t)ifunc - ((intptr_t)&p[1] + 4);
-         */
+          p[1] = (addr & 0x000000ff) >> 000;
-        if ((code & 0x0000FFFFFFFFFFFF) == 0x0000441F0F66 ||
+          p[2] = (addr & 0x0000ff00) >> 010;
-            (code & 0x0000FFFFFFFFFFFF) ==
+          p[3] = (addr & 0x00ff0000) >> 020;
-                ((((kMcount - ((intptr_t)&p[2] + 4)) << 16) | 0xE867) &
+          p[4] = (addr & 0xff000000) >> 030;
-                 0x0000FFFFFFFFFFFF) ||
-            (code & 0x0000FFFFFFFFFFFF) ==
-                ((((kMcount - ((intptr_t)&p[2] + 4)) << 16) | 0xFF15) &
-                 0x0000FFFFFFFFFFFF)) {
-          p[0] = 0x67;
-          p[1] = 0xE8;
-          addr = (intptr_t)ifunc - ((intptr_t)&p[2] + 4);
-          p[2] = (addr & 0x000000ff) >> 000;
-          p[3] = (addr & 0x0000ff00) >> 010;
-          p[4] = (addr & 0x00ff0000) >> 020;
-          p[5] = (addr & 0xff000000) >> 030;
-          break;
-        }
-
-        /*
-         * Test for -mnop-mcount (w/ -fno-pie)
-         */
-        mcode = code & 0x000000FFFFFFFFFF;
-        if ((mcode == 0x00441F0F /*   nopl 0x00(%eax,%eax,1) [canonical] */) ||
-            (mcode == 0x00041F0F67 /* nopl (%eax,%eax,1)     [older gcc] */)) {
-          if (p[-1] != 0x66 /*        nopw 0x0(%rax,%rax,1)  [donotwant] */) {
-            p[0] = 0xE8 /* call Jvds */;
-            addr = (intptr_t)ifunc - ((intptr_t)&p[1] + 4);
-            p[1] = (addr & 0x000000ff) >> 000;
-            p[2] = (addr & 0x0000ff00) >> 010;
-            p[3] = (addr & 0x00ff0000) >> 020;
-            p[4] = (addr & 0xff000000) >> 030;
-          }
-          break;
         }
+        break;
       }
     }
-    mprotect((void *)symbols->addr_base, kPrivilegedStart - symbols->addr_base,
-             PROT_READ | PROT_EXEC);
   }
-  if (!IsWindows()) {
+  __morph_end();
-    sys_sigprocmask(SIG_SETMASK, &oldmask, NULL);
+  return 0;
-  }
-  return rc;
 }

libc/runtime/morph.greg.c

@@ -0,0 +1,79 @@
+/*-*- mode:c;indent-tabs-mode:nil;c-basic-offset:2;tab-width:8;coding:utf-8 -*-│
+│vi: set net ft=c ts=2 sts=2 sw=2 fenc=utf-8                                :vi│
+╞══════════════════════════════════════════════════════════════════════════════╡
+│ Copyright 2022 Justine Alexandra Roberts Tunney                              │
+│                                                                              │
+│ Permission to use, copy, modify, and/or distribute this software for         │
+│ any purpose with or without fee is hereby granted, provided that the         │
+│ above copyright notice and this permission notice appear in all copies.      │
+│                                                                              │
+│ THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL                │
+│ WARRANTIES WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED                │
+│ WARRANTIES OF MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE             │
+│ AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL         │
+│ DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR        │
+│ PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER               │
+│ TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR             │
+│ PERFORMANCE OF THIS SOFTWARE.                                                │
+╚─────────────────────────────────────────────────────────────────────────────*/
+#define ShouldUseMsabiAttribute() 1
+#include "libc/bits/asmflag.h"
+#include "libc/calls/internal.h"
+#include "libc/calls/sigbits.h"
+#include "libc/calls/struct/sigset.h"
+#include "libc/dce.h"
+#include "libc/errno.h"
+#include "libc/intrin/kprintf.h"
+#include "libc/nt/enum/pageflags.h"
+#include "libc/nt/memory.h"
+#include "libc/nt/runtime.h"
+#include "libc/nt/thunk/msabi.h"
+#include "libc/runtime/runtime.h"
+#include "libc/sysv/consts/nr.h"
+#include "libc/sysv/consts/prot.h"
+#include "libc/sysv/consts/sig.h"
+
+__msabi extern typeof(VirtualProtect) *const __imp_VirtualProtect;
+
+static int64_t vector;
+static sigset_t oldss;
+
+static privileged void __morph_mprotect(void *addr, size_t size, int prot,
+                                        int ntprot) {
+  int ax, dx;
+  uint32_t op;
+  if (!IsWindows()) {
+    asm volatile("syscall"
+                 : "=a"(ax), "=d"(dx)
+                 : "0"(__NR_mprotect), "D"(addr), "S"(size), "1"(prot)
+                 : "rcx", "r11", "memory");
+  } else {
+    __imp_VirtualProtect(addr, size, ntprot, &op);
+  }
+}
+
+/**
+ * Begins code morphing execuatble.
+ *
+ * @return 0 on success, or -1 w/ errno
+ */
+privileged void __morph_begin(void) {
+  sigset_t ss;
+  if (!IsWindows()) {
+    sigfillset(&ss);
+    sys_sigprocmask(SIG_BLOCK, &ss, &oldss);
+  }
+  __morph_mprotect(_base, __privileged_addr - _base, PROT_READ | PROT_WRITE,
+                   kNtPageWritecopy);
+}
+
+/**
+ * Begins code morphing execuatble.
+ */
+privileged void __morph_end(void) {
+  __morph_mprotect(_base, __privileged_addr - _base, PROT_READ | PROT_EXEC,
+                   kNtPageExecuteRead);
+  if (!IsWindows()) {
+    sys_sigprocmask(SIG_SETMASK, &oldss, 0);
+  }
+}

libc/runtime/mprotect.greg.c

@@ -20,6 +20,7 @@
 #include "libc/calls/internal.h"
 #include "libc/calls/strace.internal.h"
 #include "libc/dce.h"
+#include "libc/errno.h"
 #include "libc/intrin/describeflags.internal.h"
 #include "libc/intrin/kprintf.h"
 #include "libc/runtime/internal.h"
@@ -35,15 +36,17 @@
  * @return 0 on success, or -1 w/ errno
  * @see mmap()
  */
-privileged int mprotect(void *addr, size_t size, int prot) {
+int mprotect(void *addr, size_t size, int prot) {
   int64_t rc;
   if (SupportsWindows() && (prot & ~(PROT_READ | PROT_WRITE | PROT_EXEC |
                                      PROT_GROWSDOWN | PROT_GROWSUP))) {
-    rc = einval();  // unix checks prot before checking size
+    errno = EINVAL;  // unix checks prot before checking size
+    rc = -1;
   } else if (!size) {
     return 0;  // make new technology consistent with unix
   } else if (UNLIKELY((intptr_t)addr & 4095)) {
-    rc = einval();
+    errno = EINVAL;  // unix checks prot before checking size
+    rc = -1;
   } else if (!IsWindows()) {
     rc = sys_mprotect(addr, size, prot);
   } else {

libc/runtime/runtime.h

@@ -28,10 +28,12 @@ extern unsigned char _etext[] forcealign(PAGESIZE); /* αpε */
 extern unsigned char _edata[] forcealign(PAGESIZE); /* αpε */
 extern unsigned char _ezip[];                       /* αpε */
 extern unsigned char _end[] forcealign(FRAMESIZE);  /* αpε */
-extern unsigned char _ereal;                        /* αpε */
+extern unsigned char _ereal[];                      /* αpε */
-extern unsigned char __privileged_start;            /* αpε */
+extern unsigned char __privileged_start[];          /* αpε */
-extern unsigned char __test_start;                  /* αpε */
+extern unsigned char __privileged_addr[];           /* αpε */
-extern unsigned char __ro;                          /* αpε */
+extern unsigned char __privileged_size[];           /* αpε */
+extern unsigned char __test_start[];                /* αpε */
+extern unsigned char __ro[];                        /* αpε */
 extern unsigned char *__relo_start[];               /* αpε */
 extern unsigned char *__relo_end[];                 /* αpε */
 extern uint8_t __zip_start[];                       /* αpε */
@@ -105,6 +107,10 @@ char *GetInterpreterExecutableName(char *, size_t);
 void __printargs(const char *);
 void __paginate(int, const char *);
 int __arg_max(void);
+void __morph_begin(void);
+void __morph_end(void);
+unsigned char *GetFirstInstruction(void);
+unsigned char *GetInstructionLengths(void);
 
 COSMOPOLITAN_C_END_
 #endif /* !(__ASSEMBLER__ + __LINKER__ + 0) */

libc/runtime/runtime.mk

@@ -43,7 +43,8 @@ LIBC_RUNTIME_A_DIRECTDEPS =				\
 	LIBC_STR					\
 	LIBC_STUBS					\
 	LIBC_SYSV					\
-	LIBC_SYSV_CALLS
+	LIBC_SYSV_CALLS					\
+	THIRD_PARTY_XED
 
 LIBC_RUNTIME_A_DEPS :=					\
 	$(call uniq,$(foreach x,$(LIBC_RUNTIME_A_DIRECTDEPS),$($(x))))
@@ -64,11 +65,13 @@ o/$(MODE)/libc/runtime/directmap.o			\
 o/$(MODE)/libc/runtime/directmapnt.o			\
 o/$(MODE)/libc/runtime/findmemoryinterval.o		\
 o/$(MODE)/libc/runtime/ftrace.greg.o			\
+o/$(MODE)/libc/runtime/sys_mprotect.greg.o		\
 o/$(MODE)/libc/runtime/ftracer.o			\
 o/$(MODE)/libc/runtime/ezmap.o				\
 o/$(MODE)/libc/runtime/getdosargv.o			\
 o/$(MODE)/libc/runtime/getdosenviron.o			\
 o/$(MODE)/libc/runtime/hook.greg.o			\
+o/$(MODE)/libc/runtime/morph.greg.o			\
 o/$(MODE)/libc/runtime/mprotect.greg.o			\
 o/$(MODE)/libc/runtime/mprotect-nt.greg.o		\
 o/$(MODE)/libc/runtime/ismemtracked.greg.o		\
@@ -85,6 +88,7 @@ o/$(MODE)/libc/runtime/winmain.greg.o			\
 o/$(MODE)/libc/runtime/opensymboltable.o		\
 o/$(MODE)/libc/runtime/getsymboltable.greg.o:		\
 		OVERRIDE_CFLAGS +=			\
+			-Os				\
 			-ffreestanding			\
 			$(NO_MAGIC)
 

libc/runtime/vfork.S

@@ -53,7 +53,7 @@ vfork:
 	call	__stracef
 #endif /* SYSDEBUG */
 	mov	__NR_vfork(%rip),%eax
-	mov	errno(%rip),%r8d	# avoid question of @vforksafe errno
+	mov	__errno(%rip),%r8d	# avoid question of @vforksafe errno
 	pop	%rsi			# saves return address in a register
 #if SupportsBsd()
 	testb	IsBsd()
@@ -65,7 +65,7 @@ vfork:
 	cmp	$-4095,%eax
 	jae	systemfive_error
 #endif
-0:	mov	%r8d,errno(%rip)
+0:	mov	%r8d,__errno(%rip)
 	ezlea	__vforked,di
 	test	%eax,%eax
 	jz	1f

libc/runtime/winmain.greg.c

@@ -216,7 +216,7 @@ __msabi static textwindows wontreturn void WinMainNew(const char16_t *cmdline) {
   _mmi.p[0].x = allocaddr >> 16;
   _mmi.p[0].y = (allocaddr >> 16) + ((allocsize >> 16) - 1);
   _mmi.p[0].prot = prot;
-  _mmi.p[0].flags = MAP_PRIVATE | MAP_ANONYMOUS;
+  _mmi.p[0].flags = 0x00000026;  // stack+anonymous
   _mmi.p[0].size = allocsize;
   _mmi.i = 1;
   wa = (struct WinArgs *)allocaddr;

libc/stubs/ld.S

@@ -31,6 +31,8 @@
 	_ehead = 0
 	_ereal = 0
 	__privileged_start = 0
+	__privileged_addr = 0
+	__privileged_size = 0
 	__test_start = 0
 	__ro = 0
 	__relo_start = 0
@@ -51,6 +53,8 @@
 	.globl	ape_xlm
 	.globl	__relo_start
 	.globl	__relo_end
+	.globl	__privileged_size
+	.globl	__privileged_addr
 	.globl	__privileged_start
 	.globl	__ro
 	.globl	__test_start
@@ -72,6 +76,8 @@
 	.weak	ape_xlm
 	.weak	__relo_start
 	.weak	__relo_end
+	.weak	__privileged_size
+	.weak	__privileged_addr
 	.weak	__privileged_start
 	.weak	__ro
 	.weak	__test_start

libc/sysv/errfuns/e2big.S

@@ -1,10 +1,12 @@
 #include "libc/macros.internal.h"
 .text.unlikely
 
+.section .privileged,"ax",@progbits
+
 e2big:	.leafprologue
-	.profilable
+	mov	E2BIG(%rip),%ecx
-	mov	E2BIG(%rip),%eax
+	.errno
-	mov	%eax,errno(%rip)
+	mov	%ecx,(%rax)
 	push	$-1
 	pop	%rax
 	.leafepilogue

libc/sysv/errfuns/eacces.S

@@ -1,10 +1,12 @@
 #include "libc/macros.internal.h"
 .text.unlikely
 
+.section .privileged,"ax",@progbits
+
 eacces:	.leafprologue
-	.profilable
+	mov	EACCES(%rip),%ecx
-	mov	EACCES(%rip),%eax
+	.errno
-	mov	%eax,errno(%rip)
+	mov	%ecx,(%rax)
 	push	$-1
 	pop	%rax
 	.leafepilogue

libc/sysv/errfuns/eaddrinuse.S

@@ -1,11 +1,13 @@
 #include "libc/macros.internal.h"
 .text.unlikely
 
+.section .privileged,"ax",@progbits
+
 eaddrinuse:
 	.leafprologue
-	.profilable
+	mov	EADDRINUSE(%rip),%ecx
-	mov	EADDRINUSE(%rip),%eax
+	.errno
-	mov	%eax,errno(%rip)
+	mov	%ecx,(%rax)
 	push	$-1
 	pop	%rax
 	.leafepilogue

libc/sysv/errfuns/eaddrnotavail.S

@@ -1,11 +1,13 @@
 #include "libc/macros.internal.h"
 .text.unlikely
 
+.section .privileged,"ax",@progbits
+
 eaddrnotavail:
 	.leafprologue
-	.profilable
+	mov	EADDRNOTAVAIL(%rip),%ecx
-	mov	EADDRNOTAVAIL(%rip),%eax
+	.errno
-	mov	%eax,errno(%rip)
+	mov	%ecx,(%rax)
 	push	$-1
 	pop	%rax
 	.leafepilogue

libc/sysv/errfuns/eadv.S

@@ -1,10 +1,12 @@
 #include "libc/macros.internal.h"
 .text.unlikely
 
+.section .privileged,"ax",@progbits
+
 eadv:	.leafprologue
-	.profilable
+	mov	EADV(%rip),%ecx
-	mov	EADV(%rip),%eax
+	.errno
-	mov	%eax,errno(%rip)
+	mov	%ecx,(%rax)
 	push	$-1
 	pop	%rax
 	.leafepilogue

libc/sysv/errfuns/eafnosupport.S

@@ -1,11 +1,13 @@
 #include "libc/macros.internal.h"
 .text.unlikely
 
+.section .privileged,"ax",@progbits
+
 eafnosupport:
 	.leafprologue
-	.profilable
+	mov	EAFNOSUPPORT(%rip),%ecx
-	mov	EAFNOSUPPORT(%rip),%eax
+	.errno
-	mov	%eax,errno(%rip)
+	mov	%ecx,(%rax)
 	push	$-1
 	pop	%rax
 	.leafepilogue

libc/sysv/errfuns/eagain.S

@@ -1,10 +1,12 @@
 #include "libc/macros.internal.h"
 .text.unlikely
 
+.section .privileged,"ax",@progbits
+
 eagain:	.leafprologue
-	.profilable
+	mov	EAGAIN(%rip),%ecx
-	mov	EAGAIN(%rip),%eax
+	.errno
-	mov	%eax,errno(%rip)
+	mov	%ecx,(%rax)
 	push	$-1
 	pop	%rax
 	.leafepilogue

libc/sysv/errfuns/ealready.S

@@ -1,11 +1,13 @@
 #include "libc/macros.internal.h"
 .text.unlikely
 
+.section .privileged,"ax",@progbits
+
 ealready:
 	.leafprologue
-	.profilable
+	mov	EALREADY(%rip),%ecx
-	mov	EALREADY(%rip),%eax
+	.errno
-	mov	%eax,errno(%rip)
+	mov	%ecx,(%rax)
 	push	$-1
 	pop	%rax
 	.leafepilogue

libc/sysv/errfuns/ebade.S

@@ -1,10 +1,12 @@
 #include "libc/macros.internal.h"
 .text.unlikely
 
+.section .privileged,"ax",@progbits
+
 ebade:	.leafprologue
-	.profilable
+	mov	EBADE(%rip),%ecx
-	mov	EBADE(%rip),%eax
+	.errno
-	mov	%eax,errno(%rip)
+	mov	%ecx,(%rax)
 	push	$-1
 	pop	%rax
 	.leafepilogue

libc/sysv/errfuns/ebadf.S

@@ -1,10 +1,12 @@
 #include "libc/macros.internal.h"
 .text.unlikely
 
+.section .privileged,"ax",@progbits
+
 ebadf:	.leafprologue
-	.profilable
+	mov	EBADF(%rip),%ecx
-	mov	EBADF(%rip),%eax
+	.errno
-	mov	%eax,errno(%rip)
+	mov	%ecx,(%rax)
 	push	$-1
 	pop	%rax
 	.leafepilogue

libc/sysv/errfuns/ebadfd.S

@@ -1,10 +1,12 @@
 #include "libc/macros.internal.h"
 .text.unlikely
 
+.section .privileged,"ax",@progbits
+
 ebadfd:	.leafprologue
-	.profilable
+	mov	EBADFD(%rip),%ecx
-	mov	EBADFD(%rip),%eax
+	.errno
-	mov	%eax,errno(%rip)
+	mov	%ecx,(%rax)
 	push	$-1
 	pop	%rax
 	.leafepilogue

libc/sysv/errfuns/ebadmsg.S

@@ -1,11 +1,13 @@
 #include "libc/macros.internal.h"
 .text.unlikely
 
+.section .privileged,"ax",@progbits
+
 ebadmsg:
 	.leafprologue
-	.profilable
+	mov	EBADMSG(%rip),%ecx
-	mov	EBADMSG(%rip),%eax
+	.errno
-	mov	%eax,errno(%rip)
+	mov	%ecx,(%rax)
 	push	$-1
 	pop	%rax
 	.leafepilogue

libc/sysv/errfuns/ebadr.S

@@ -1,10 +1,12 @@
 #include "libc/macros.internal.h"
 .text.unlikely
 
+.section .privileged,"ax",@progbits
+
 ebadr:	.leafprologue
-	.profilable
+	mov	EBADR(%rip),%ecx
-	mov	EBADR(%rip),%eax
+	.errno
-	mov	%eax,errno(%rip)
+	mov	%ecx,(%rax)
 	push	$-1
 	pop	%rax
 	.leafepilogue

libc/sysv/errfuns/ebadrqc.S

@@ -1,11 +1,13 @@
 #include "libc/macros.internal.h"
 .text.unlikely
 
+.section .privileged,"ax",@progbits
+
 ebadrqc:
 	.leafprologue
-	.profilable
+	mov	EBADRQC(%rip),%ecx
-	mov	EBADRQC(%rip),%eax
+	.errno
-	mov	%eax,errno(%rip)
+	mov	%ecx,(%rax)
 	push	$-1
 	pop	%rax
 	.leafepilogue

libc/sysv/errfuns/ebadslt.S

@@ -1,11 +1,13 @@
 #include "libc/macros.internal.h"
 .text.unlikely
 
+.section .privileged,"ax",@progbits
+
 ebadslt:
 	.leafprologue
-	.profilable
+	mov	EBADSLT(%rip),%ecx
-	mov	EBADSLT(%rip),%eax
+	.errno
-	mov	%eax,errno(%rip)
+	mov	%ecx,(%rax)
 	push	$-1
 	pop	%rax
 	.leafepilogue

libc/sysv/errfuns/ebusy.S

@@ -1,10 +1,12 @@
 #include "libc/macros.internal.h"
 .text.unlikely
 
+.section .privileged,"ax",@progbits
+
 ebusy:	.leafprologue
-	.profilable
+	mov	EBUSY(%rip),%ecx
-	mov	EBUSY(%rip),%eax
+	.errno
-	mov	%eax,errno(%rip)
+	mov	%ecx,(%rax)
 	push	$-1
 	pop	%rax
 	.leafepilogue

libc/sysv/errfuns/ecanceled.S

@@ -1,11 +1,13 @@
 #include "libc/macros.internal.h"
 .text.unlikely
 
+.section .privileged,"ax",@progbits
+
 ecanceled:
 	.leafprologue
-	.profilable
+	mov	ECANCELED(%rip),%ecx
-	mov	ECANCELED(%rip),%eax
+	.errno
-	mov	%eax,errno(%rip)
+	mov	%ecx,(%rax)
 	push	$-1
 	pop	%rax
 	.leafepilogue

libc/sysv/errfuns/echild.S

@@ -1,10 +1,12 @@
 #include "libc/macros.internal.h"
 .text.unlikely
 
+.section .privileged,"ax",@progbits
+
 echild:	.leafprologue
-	.profilable
+	mov	ECHILD(%rip),%ecx
-	mov	ECHILD(%rip),%eax
+	.errno
-	mov	%eax,errno(%rip)
+	mov	%ecx,(%rax)
 	push	$-1
 	pop	%rax
 	.leafepilogue

libc/sysv/errfuns/echrng.S

@@ -1,10 +1,12 @@
 #include "libc/macros.internal.h"
 .text.unlikely
 
+.section .privileged,"ax",@progbits
+
 echrng:	.leafprologue
-	.profilable
+	mov	ECHRNG(%rip),%ecx
-	mov	ECHRNG(%rip),%eax
+	.errno
-	mov	%eax,errno(%rip)
+	mov	%ecx,(%rax)
 	push	$-1
 	pop	%rax
 	.leafepilogue

libc/sysv/errfuns/ecomm.S

@@ -1,10 +1,12 @@
 #include "libc/macros.internal.h"
 .text.unlikely
 
+.section .privileged,"ax",@progbits
+
 ecomm:	.leafprologue
-	.profilable
+	mov	ECOMM(%rip),%ecx
-	mov	ECOMM(%rip),%eax
+	.errno
-	mov	%eax,errno(%rip)
+	mov	%ecx,(%rax)
 	push	$-1
 	pop	%rax
 	.leafepilogue

libc/sysv/errfuns/econnaborted.S

@@ -1,11 +1,13 @@
 #include "libc/macros.internal.h"
 .text.unlikely
 
+.section .privileged,"ax",@progbits
+
 econnaborted:
 	.leafprologue
-	.profilable
+	mov	ECONNABORTED(%rip),%ecx
-	mov	ECONNABORTED(%rip),%eax
+	.errno
-	mov	%eax,errno(%rip)
+	mov	%ecx,(%rax)
 	push	$-1
 	pop	%rax
 	.leafepilogue

libc/sysv/errfuns/econnrefused.S

@@ -1,11 +1,13 @@
 #include "libc/macros.internal.h"
 .text.unlikely
 
+.section .privileged,"ax",@progbits
+
 econnrefused:
 	.leafprologue
-	.profilable
+	mov	ECONNREFUSED(%rip),%ecx
-	mov	ECONNREFUSED(%rip),%eax
+	.errno
-	mov	%eax,errno(%rip)
+	mov	%ecx,(%rax)
 	push	$-1
 	pop	%rax
 	.leafepilogue

libc/sysv/errfuns/econnreset.S

@@ -1,11 +1,13 @@
 #include "libc/macros.internal.h"
 .text.unlikely
 
+.section .privileged,"ax",@progbits
+
 econnreset:
 	.leafprologue
-	.profilable
+	mov	ECONNRESET(%rip),%ecx
-	mov	ECONNRESET(%rip),%eax
+	.errno
-	mov	%eax,errno(%rip)
+	mov	%ecx,(%rax)
 	push	$-1
 	pop	%rax
 	.leafepilogue

libc/sysv/errfuns/edeadlk.S

@@ -1,11 +1,13 @@
 #include "libc/macros.internal.h"
 .text.unlikely
 
+.section .privileged,"ax",@progbits
+
 edeadlk:
 	.leafprologue
-	.profilable
+	mov	EDEADLK(%rip),%ecx
-	mov	EDEADLK(%rip),%eax
+	.errno
-	mov	%eax,errno(%rip)
+	mov	%ecx,(%rax)
 	push	$-1
 	pop	%rax
 	.leafepilogue

libc/sysv/errfuns/edestaddrreq.S

@@ -1,11 +1,13 @@
 #include "libc/macros.internal.h"
 .text.unlikely
 
+.section .privileged,"ax",@progbits
+
 edestaddrreq:
 	.leafprologue
-	.profilable
+	mov	EDESTADDRREQ(%rip),%ecx
-	mov	EDESTADDRREQ(%rip),%eax
+	.errno
-	mov	%eax,errno(%rip)
+	mov	%ecx,(%rax)
 	push	$-1
 	pop	%rax
 	.leafepilogue

libc/sysv/errfuns/edom.S

@@ -1,10 +1,12 @@
 #include "libc/macros.internal.h"
 .text.unlikely
 
+.section .privileged,"ax",@progbits
+
 edom:	.leafprologue
-	.profilable
+	mov	EDOM(%rip),%ecx
-	mov	EDOM(%rip),%eax
+	.errno
-	mov	%eax,errno(%rip)
+	mov	%ecx,(%rax)
 	push	$-1
 	pop	%rax
 	.leafepilogue

libc/sysv/errfuns/edotdot.S

@@ -1,11 +1,13 @@
 #include "libc/macros.internal.h"
 .text.unlikely
 
+.section .privileged,"ax",@progbits
+
 edotdot:
 	.leafprologue
-	.profilable
+	mov	EDOTDOT(%rip),%ecx
-	mov	EDOTDOT(%rip),%eax
+	.errno
-	mov	%eax,errno(%rip)
+	mov	%ecx,(%rax)
 	push	$-1
 	pop	%rax
 	.leafepilogue

libc/sysv/errfuns/edquot.S

@@ -1,10 +1,12 @@
 #include "libc/macros.internal.h"
 .text.unlikely
 
+.section .privileged,"ax",@progbits
+
 edquot:	.leafprologue
-	.profilable
+	mov	EDQUOT(%rip),%ecx
-	mov	EDQUOT(%rip),%eax
+	.errno
-	mov	%eax,errno(%rip)
+	mov	%ecx,(%rax)
 	push	$-1
 	pop	%rax
 	.leafepilogue

libc/sysv/errfuns/eexist.S

@@ -1,10 +1,12 @@
 #include "libc/macros.internal.h"
 .text.unlikely
 
+.section .privileged,"ax",@progbits
+
 eexist:	.leafprologue
-	.profilable
+	mov	EEXIST(%rip),%ecx
-	mov	EEXIST(%rip),%eax
+	.errno
-	mov	%eax,errno(%rip)
+	mov	%ecx,(%rax)
 	push	$-1
 	pop	%rax
 	.leafepilogue

libc/sysv/errfuns/efault.S

@@ -1,10 +1,12 @@
 #include "libc/macros.internal.h"
 .text.unlikely
 
+.section .privileged,"ax",@progbits
+
 efault:	.leafprologue
-	.profilable
+	mov	EFAULT(%rip),%ecx
-	mov	EFAULT(%rip),%eax
+	.errno
-	mov	%eax,errno(%rip)
+	mov	%ecx,(%rax)
 	push	$-1
 	pop	%rax
 	.leafepilogue

libc/sysv/errfuns/efbig.S

@@ -1,10 +1,12 @@
 #include "libc/macros.internal.h"
 .text.unlikely
 
+.section .privileged,"ax",@progbits
+
 efbig:	.leafprologue
-	.profilable
+	mov	EFBIG(%rip),%ecx
-	mov	EFBIG(%rip),%eax
+	.errno
-	mov	%eax,errno(%rip)
+	mov	%ecx,(%rax)
 	push	$-1
 	pop	%rax
 	.leafepilogue

libc/sysv/errfuns/ehostdown.S

@@ -1,11 +1,13 @@
 #include "libc/macros.internal.h"
 .text.unlikely
 
+.section .privileged,"ax",@progbits
+
 ehostdown:
 	.leafprologue
-	.profilable
+	mov	EHOSTDOWN(%rip),%ecx
-	mov	EHOSTDOWN(%rip),%eax
+	.errno
-	mov	%eax,errno(%rip)
+	mov	%ecx,(%rax)
 	push	$-1
 	pop	%rax
 	.leafepilogue

libc/sysv/errfuns/ehostunreach.S

@@ -1,11 +1,13 @@
 #include "libc/macros.internal.h"
 .text.unlikely
 
+.section .privileged,"ax",@progbits
+
 ehostunreach:
 	.leafprologue
-	.profilable
+	mov	EHOSTUNREACH(%rip),%ecx
-	mov	EHOSTUNREACH(%rip),%eax
+	.errno
-	mov	%eax,errno(%rip)
+	mov	%ecx,(%rax)
 	push	$-1
 	pop	%rax
 	.leafepilogue

libc/sysv/errfuns/ehwpoison.S

@@ -1,11 +1,13 @@
 #include "libc/macros.internal.h"
 .text.unlikely
 
+.section .privileged,"ax",@progbits
+
 ehwpoison:
 	.leafprologue
-	.profilable
+	mov	EHWPOISON(%rip),%ecx
-	mov	EHWPOISON(%rip),%eax
+	.errno
-	mov	%eax,errno(%rip)
+	mov	%ecx,(%rax)
 	push	$-1
 	pop	%rax
 	.leafepilogue

libc/sysv/errfuns/eidrm.S

@@ -1,10 +1,12 @@
 #include "libc/macros.internal.h"
 .text.unlikely
 
+.section .privileged,"ax",@progbits
+
 eidrm:	.leafprologue
-	.profilable
+	mov	EIDRM(%rip),%ecx
-	mov	EIDRM(%rip),%eax
+	.errno
-	mov	%eax,errno(%rip)
+	mov	%ecx,(%rax)
 	push	$-1
 	pop	%rax
 	.leafepilogue

libc/sysv/errfuns/eilseq.S

@@ -1,10 +1,12 @@
 #include "libc/macros.internal.h"
 .text.unlikely
 
+.section .privileged,"ax",@progbits
+
 eilseq:	.leafprologue
-	.profilable
+	mov	EILSEQ(%rip),%ecx
-	mov	EILSEQ(%rip),%eax
+	.errno
-	mov	%eax,errno(%rip)
+	mov	%ecx,(%rax)
 	push	$-1
 	pop	%rax
 	.leafepilogue

libc/sysv/errfuns/einprogress.S

@@ -1,11 +1,13 @@
 #include "libc/macros.internal.h"
 .text.unlikely
 
+.section .privileged,"ax",@progbits
+
 einprogress:
 	.leafprologue
-	.profilable
+	mov	EINPROGRESS(%rip),%ecx
-	mov	EINPROGRESS(%rip),%eax
+	.errno
-	mov	%eax,errno(%rip)
+	mov	%ecx,(%rax)
 	push	$-1
 	pop	%rax
 	.leafepilogue

libc/sysv/errfuns/eintr.S

@@ -1,10 +1,12 @@
 #include "libc/macros.internal.h"
 .text.unlikely
 
+.section .privileged,"ax",@progbits
+
 eintr:	.leafprologue
-	.profilable
+	mov	EINTR(%rip),%ecx
-	mov	EINTR(%rip),%eax
+	.errno
-	mov	%eax,errno(%rip)
+	mov	%ecx,(%rax)
 	push	$-1
 	pop	%rax
 	.leafepilogue

libc/sysv/errfuns/einval.S

@@ -1,10 +1,12 @@
 #include "libc/macros.internal.h"
 .text.unlikely
 
+.section .privileged,"ax",@progbits
+
 einval:	.leafprologue
-	.profilable
+	mov	EINVAL(%rip),%ecx
-	mov	EINVAL(%rip),%eax
+	.errno
-	mov	%eax,errno(%rip)
+	mov	%ecx,(%rax)
 	push	$-1
 	pop	%rax
 	.leafepilogue

libc/sysv/errfuns/eio.S

@@ -1,10 +1,12 @@
 #include "libc/macros.internal.h"
 .text.unlikely
 
+.section .privileged,"ax",@progbits
+
 eio:	.leafprologue
-	.profilable
+	mov	EIO(%rip),%ecx
-	mov	EIO(%rip),%eax
+	.errno
-	mov	%eax,errno(%rip)
+	mov	%ecx,(%rax)
 	push	$-1
 	pop	%rax
 	.leafepilogue

libc/sysv/errfuns/eisconn.S

@@ -1,11 +1,13 @@
 #include "libc/macros.internal.h"
 .text.unlikely
 
+.section .privileged,"ax",@progbits
+
 eisconn:
 	.leafprologue
-	.profilable
+	mov	EISCONN(%rip),%ecx
-	mov	EISCONN(%rip),%eax
+	.errno
-	mov	%eax,errno(%rip)
+	mov	%ecx,(%rax)
 	push	$-1
 	pop	%rax
 	.leafepilogue

libc/sysv/errfuns/eisdir.S

@@ -1,10 +1,12 @@
 #include "libc/macros.internal.h"
 .text.unlikely
 
+.section .privileged,"ax",@progbits
+
 eisdir:	.leafprologue
-	.profilable
+	mov	EISDIR(%rip),%ecx
-	mov	EISDIR(%rip),%eax
+	.errno
-	mov	%eax,errno(%rip)
+	mov	%ecx,(%rax)
 	push	$-1
 	pop	%rax
 	.leafepilogue

libc/sysv/errfuns/eisnam.S

@@ -1,10 +1,12 @@
 #include "libc/macros.internal.h"
 .text.unlikely
 
+.section .privileged,"ax",@progbits
+
 eisnam:	.leafprologue
-	.profilable
+	mov	EISNAM(%rip),%ecx
-	mov	EISNAM(%rip),%eax
+	.errno
-	mov	%eax,errno(%rip)
+	mov	%ecx,(%rax)
 	push	$-1
 	pop	%rax
 	.leafepilogue

libc/sysv/errfuns/ekeyexpired.S

@@ -1,11 +1,13 @@
 #include "libc/macros.internal.h"
 .text.unlikely
 
+.section .privileged,"ax",@progbits
+
 ekeyexpired:
 	.leafprologue
-	.profilable
+	mov	EKEYEXPIRED(%rip),%ecx
-	mov	EKEYEXPIRED(%rip),%eax
+	.errno
-	mov	%eax,errno(%rip)
+	mov	%ecx,(%rax)
 	push	$-1
 	pop	%rax
 	.leafepilogue

libc/sysv/errfuns/ekeyrejected.S

@@ -1,11 +1,13 @@
 #include "libc/macros.internal.h"
 .text.unlikely
 
+.section .privileged,"ax",@progbits
+
 ekeyrejected:
 	.leafprologue
-	.profilable
+	mov	EKEYREJECTED(%rip),%ecx
-	mov	EKEYREJECTED(%rip),%eax
+	.errno
-	mov	%eax,errno(%rip)
+	mov	%ecx,(%rax)
 	push	$-1
 	pop	%rax
 	.leafepilogue

libc/sysv/errfuns/ekeyrevoked.S

@@ -1,11 +1,13 @@
 #include "libc/macros.internal.h"
 .text.unlikely
 
+.section .privileged,"ax",@progbits
+
 ekeyrevoked:
 	.leafprologue
-	.profilable
+	mov	EKEYREVOKED(%rip),%ecx
-	mov	EKEYREVOKED(%rip),%eax
+	.errno
-	mov	%eax,errno(%rip)
+	mov	%ecx,(%rax)
 	push	$-1
 	pop	%rax
 	.leafepilogue

libc/sysv/errfuns/el2hlt.S

@@ -1,10 +1,12 @@
 #include "libc/macros.internal.h"
 .text.unlikely
 
+.section .privileged,"ax",@progbits
+
 el2hlt:	.leafprologue
-	.profilable
+	mov	EL2HLT(%rip),%ecx
-	mov	EL2HLT(%rip),%eax
+	.errno
-	mov	%eax,errno(%rip)
+	mov	%ecx,(%rax)
 	push	$-1
 	pop	%rax
 	.leafepilogue

libc/sysv/errfuns/el2nsync.S

@@ -1,11 +1,13 @@
 #include "libc/macros.internal.h"
 .text.unlikely
 
+.section .privileged,"ax",@progbits
+
 el2nsync:
 	.leafprologue
-	.profilable
+	mov	EL2NSYNC(%rip),%ecx
-	mov	EL2NSYNC(%rip),%eax
+	.errno
-	mov	%eax,errno(%rip)
+	mov	%ecx,(%rax)
 	push	$-1
 	pop	%rax
 	.leafepilogue

libc/sysv/errfuns/el3hlt.S

@@ -1,10 +1,12 @@
 #include "libc/macros.internal.h"
 .text.unlikely
 
+.section .privileged,"ax",@progbits
+
 el3hlt:	.leafprologue
-	.profilable
+	mov	EL3HLT(%rip),%ecx
-	mov	EL3HLT(%rip),%eax
+	.errno
-	mov	%eax,errno(%rip)
+	mov	%ecx,(%rax)
 	push	$-1
 	pop	%rax
 	.leafepilogue

libc/sysv/errfuns/el3rst.S

@@ -1,10 +1,12 @@
 #include "libc/macros.internal.h"
 .text.unlikely
 
+.section .privileged,"ax",@progbits
+
 el3rst:	.leafprologue
-	.profilable
+	mov	EL3RST(%rip),%ecx
-	mov	EL3RST(%rip),%eax
+	.errno
-	mov	%eax,errno(%rip)
+	mov	%ecx,(%rax)
 	push	$-1
 	pop	%rax
 	.leafepilogue

libc/sysv/errfuns/elibacc.S

@@ -1,11 +1,13 @@
 #include "libc/macros.internal.h"
 .text.unlikely
 
+.section .privileged,"ax",@progbits
+
 elibacc:
 	.leafprologue
-	.profilable
+	mov	ELIBACC(%rip),%ecx
-	mov	ELIBACC(%rip),%eax
+	.errno
-	mov	%eax,errno(%rip)
+	mov	%ecx,(%rax)
 	push	$-1
 	pop	%rax
 	.leafepilogue

libc/sysv/errfuns/elibbad.S

@@ -1,11 +1,13 @@
 #include "libc/macros.internal.h"
 .text.unlikely
 
+.section .privileged,"ax",@progbits
+
 elibbad:
 	.leafprologue
-	.profilable
+	mov	ELIBBAD(%rip),%ecx
-	mov	ELIBBAD(%rip),%eax
+	.errno
-	mov	%eax,errno(%rip)
+	mov	%ecx,(%rax)
 	push	$-1
 	pop	%rax
 	.leafepilogue

libc/sysv/errfuns/elibexec.S

@@ -1,11 +1,13 @@
 #include "libc/macros.internal.h"
 .text.unlikely
 
+.section .privileged,"ax",@progbits
+
 elibexec:
 	.leafprologue
-	.profilable
+	mov	ELIBEXEC(%rip),%ecx
-	mov	ELIBEXEC(%rip),%eax
+	.errno
-	mov	%eax,errno(%rip)
+	mov	%ecx,(%rax)
 	push	$-1
 	pop	%rax
 	.leafepilogue

libc/sysv/errfuns/elibmax.S

@@ -1,11 +1,13 @@
 #include "libc/macros.internal.h"
 .text.unlikely
 
+.section .privileged,"ax",@progbits
+
 elibmax:
 	.leafprologue
-	.profilable
+	mov	ELIBMAX(%rip),%ecx
-	mov	ELIBMAX(%rip),%eax
+	.errno
-	mov	%eax,errno(%rip)
+	mov	%ecx,(%rax)
 	push	$-1
 	pop	%rax
 	.leafepilogue

libc/sysv/errfuns/elibscn.S

@@ -1,11 +1,13 @@
 #include "libc/macros.internal.h"
 .text.unlikely
 
+.section .privileged,"ax",@progbits
+
 elibscn:
 	.leafprologue
-	.profilable
+	mov	ELIBSCN(%rip),%ecx
-	mov	ELIBSCN(%rip),%eax
+	.errno
-	mov	%eax,errno(%rip)
+	mov	%ecx,(%rax)
 	push	$-1
 	pop	%rax
 	.leafepilogue

libc/sysv/errfuns/elnrng.S

@@ -1,10 +1,12 @@
 #include "libc/macros.internal.h"
 .text.unlikely
 
+.section .privileged,"ax",@progbits
+
 elnrng:	.leafprologue
-	.profilable
+	mov	ELNRNG(%rip),%ecx
-	mov	ELNRNG(%rip),%eax
+	.errno
-	mov	%eax,errno(%rip)
+	mov	%ecx,(%rax)
 	push	$-1
 	pop	%rax
 	.leafepilogue

libc/sysv/errfuns/eloop.S

@@ -1,10 +1,12 @@
 #include "libc/macros.internal.h"
 .text.unlikely
 
+.section .privileged,"ax",@progbits
+
 eloop:	.leafprologue
-	.profilable
+	mov	ELOOP(%rip),%ecx
-	mov	ELOOP(%rip),%eax
+	.errno
-	mov	%eax,errno(%rip)
+	mov	%ecx,(%rax)
 	push	$-1
 	pop	%rax
 	.leafepilogue

libc/sysv/errfuns/emediumtype.S

@@ -1,11 +1,13 @@
 #include "libc/macros.internal.h"
 .text.unlikely
 
+.section .privileged,"ax",@progbits
+
 emediumtype:
 	.leafprologue
-	.profilable
+	mov	EMEDIUMTYPE(%rip),%ecx
-	mov	EMEDIUMTYPE(%rip),%eax
+	.errno
-	mov	%eax,errno(%rip)
+	mov	%ecx,(%rax)
 	push	$-1
 	pop	%rax
 	.leafepilogue

libc/sysv/errfuns/emfile.S

@@ -1,10 +1,12 @@
 #include "libc/macros.internal.h"
 .text.unlikely
 
+.section .privileged,"ax",@progbits
+
 emfile:	.leafprologue
-	.profilable
+	mov	EMFILE(%rip),%ecx
-	mov	EMFILE(%rip),%eax
+	.errno
-	mov	%eax,errno(%rip)
+	mov	%ecx,(%rax)
 	push	$-1
 	pop	%rax
 	.leafepilogue

libc/sysv/errfuns/emlink.S

@@ -1,10 +1,12 @@
 #include "libc/macros.internal.h"
 .text.unlikely
 
+.section .privileged,"ax",@progbits
+
 emlink:	.leafprologue
-	.profilable
+	mov	EMLINK(%rip),%ecx
-	mov	EMLINK(%rip),%eax
+	.errno
-	mov	%eax,errno(%rip)
+	mov	%ecx,(%rax)
 	push	$-1
 	pop	%rax
 	.leafepilogue