From ec2cb880580e3fc47f08b7bcd26e645094b7f6cf Mon Sep 17 00:00:00 2001 From: Justine Tunney Date: Thu, 19 May 2022 16:57:49 -0700 Subject: [PATCH] Make fixes and improvements - Document more compiler flags - Expose new __print_maps() api - Better overflow checking in mmap() - Improve the shell example somewhat - Fix minor runtime bugs regarding stacks - Make kill() on fork()+execve()'d children work - Support CLONE_CHILD_CLEARTID for proper joining - Fix recent possible deadlock regression with --ftrace --- ape/ape.lds | 8 +- examples/compress.c | 41 ++- examples/decompress.c | 25 +- examples/greenbean.c | 5 +- examples/shell.c | 145 ++++++++-- libc/calls/kill-nt.c | 30 +- libc/calls/sigaltstack.c | 10 +- libc/calls/wincrash.c | 1 + libc/fmt/fmt.mk | 10 + libc/fmt/{strerror_r.greg.c => strerror_r.c} | 0 libc/integral/c.inc | 4 +- libc/intrin/_spinlock_debug_1.c | 56 ++++ libc/intrin/_spinlock_debug_4.c | 57 ++++ libc/intrin/asan.c | 22 +- libc/intrin/getmagnumstr.greg.c | 2 +- libc/intrin/gettid.greg.c | 7 + libc/intrin/intrin.mk | 12 +- libc/{fmt => intrin}/kerrnodocs.S | 0 libc/{fmt => intrin}/kerrnonames.S | 0 libc/intrin/kprintf.greg.c | 14 +- libc/intrin/lockcmpxchgp.h | 24 ++ libc/intrin/spinlock.h | 100 +++---- libc/{fmt => intrin}/strerdoc.greg.c | 2 +- libc/{fmt => intrin}/strerrno.greg.c | 2 +- libc/{fmt => intrin}/strerror_wr.greg.c | 2 +- libc/intrin/tls.greg.c | 14 + libc/log/checkfail.c | 2 +- libc/log/internal.h | 2 +- libc/log/leaks.c | 2 +- libc/log/oncrash.c | 14 +- libc/log/showcrashreports.c | 65 +++-- libc/nexgen32e/checkstackalign.S | 4 +- libc/nexgen32e/nexgen32e.h | 2 +- libc/nexgen32e/nt2sysv.S | 2 + libc/nt/enum/th32cs.h | 11 + libc/nt/kernel32/CreateToolhelp32Snapshot.s | 12 +- libc/nt/kernel32/Process32FirstW.s | 12 +- libc/nt/kernel32/Process32NextW.s | 12 +- libc/nt/master.sh | 6 +- libc/nt/process.h | 5 + libc/nt/struct/processentry32.h | 19 ++ libc/runtime/{clone.greg.c => clone.c} | 262 +++++++++++------- libc/runtime/ftracer.c | 55 +++- ...getsymboltable.greg.c => getsymboltable.c} | 32 ++- libc/runtime/memtrack.greg.c | 10 +- libc/runtime/memtrack.internal.h | 3 +- libc/runtime/mmap.c | 171 +++++++----- libc/runtime/printargs.greg.c | 2 +- libc/runtime/printmaps.c | 30 ++ libc/runtime/runtime.h | 1 + libc/runtime/runtime.mk | 4 +- libc/runtime/stack.h | 33 ++- libc/runtime/winthreadlaunch.S | 46 +++ libc/stdio/unlocked/stdio_unlock.S | 1 + libc/str/str.mk | 8 + libc/str/strsignal.greg.c | 4 +- libc/testlib/quota.c | 4 +- test/libc/calls/sigaction_test.c | 8 +- test/libc/log/backtrace_test.c | 1 + test/libc/rand/rand64_test.c | 32 +-- test/libc/runtime/clone_test.c | 55 ++-- test/libc/runtime/mmap_test.c | 57 +++- test/libc/runtime/munmap_test.c | 8 +- third_party/dlmalloc/dlmalloc.mk | 2 +- .../dlmalloc/{vespene.greg.c => vespene.c} | 0 third_party/lua/lrepl.c | 2 +- tool/build/runit.c | 4 +- tool/net/redbean.c | 44 +-- 68 files changed, 1211 insertions(+), 431 deletions(-) rename libc/fmt/{strerror_r.greg.c => strerror_r.c} (100%) create mode 100644 libc/intrin/_spinlock_debug_1.c create mode 100644 libc/intrin/_spinlock_debug_4.c rename libc/{fmt => intrin}/kerrnodocs.S (100%) rename libc/{fmt => intrin}/kerrnonames.S (100%) create mode 100644 libc/intrin/lockcmpxchgp.h rename libc/{fmt => intrin}/strerdoc.greg.c (98%) rename libc/{fmt => intrin}/strerrno.greg.c (98%) rename libc/{fmt => intrin}/strerror_wr.greg.c (97%) create mode 100644 libc/nt/enum/th32cs.h create mode 100644 libc/nt/struct/processentry32.h rename libc/runtime/{clone.greg.c => clone.c} (68%) rename libc/runtime/{getsymboltable.greg.c => getsymboltable.c} (88%) create mode 100644 libc/runtime/printmaps.c create mode 100644 libc/runtime/winthreadlaunch.S rename third_party/dlmalloc/{vespene.greg.c => vespene.c} (100%) diff --git a/ape/ape.lds b/ape/ape.lds index caac90617..4e00ad30d 100644 --- a/ape/ape.lds +++ b/ape/ape.lds @@ -510,7 +510,7 @@ HIDDEN(ape_ram_rva = RVA(ape_ram_vaddr)); HIDDEN(ape_stack_pf = DEFINED(ape_stack_pf) ? ape_stack_pf : PF_R | PF_W); HIDDEN(ape_stack_prot = _PF2PROT(ape_stack_pf)); HIDDEN(ape_stack_offset = ape_ram_offset + ape_ram_filesz); -HIDDEN(ape_stack_vaddr = DEFINED(ape_stack_vaddr) ? ape_stack_vaddr : 0x700000000000 - STACKSIZE); +HIDDEN(ape_stack_vaddr = DEFINED(ape_stack_vaddr) ? ape_stack_vaddr : 0x700000000000); HIDDEN(ape_stack_paddr = ape_ram_paddr + ape_ram_filesz); HIDDEN(ape_stack_filesz = 0); HIDDEN(ape_stack_memsz = DEFINED(ape_stack_memsz) ? ape_stack_memsz : STACKSIZE); @@ -697,6 +697,12 @@ ASSERT(DEFINED(_start) || DEFINED(_start16), ASSERT(!DEFINED(_start16) || REAL(_end) < 65536, "ape won't support non-tiny real mode programs"); +ASSERT(IS2POW(ape_stack_memsz), + "ape_stack_memsz must be a two power"); + +ASSERT(!(ape_stack_vaddr & (ape_stack_memsz - 1)), + "ape_stack_vaddr must have ape_stack_memsz alignment; try using STATIC_STACK_ADDR(0x700000000000 - ape_stack_memsz);"); + /* Let's not be like Knight Capital. */ /* NOCROSSREFS_TO(.test .text) */ diff --git a/examples/compress.c b/examples/compress.c index 800e9532f..640672b66 100644 --- a/examples/compress.c +++ b/examples/compress.c @@ -9,6 +9,7 @@ #endif #include "libc/assert.h" #include "libc/errno.h" +#include "libc/fmt/conv.h" #include "libc/log/check.h" #include "libc/mem/mem.h" #include "libc/runtime/gc.internal.h" @@ -16,13 +17,36 @@ #include "libc/str/str.h" #include "third_party/zlib/zlib.h" -#define CHUNK 4096 +#define CHUNK 32768 // clang-format off // make -j8 o//examples && dd if=/dev/urandom count=100 | tee a | o//examples/compress.com | o//examples/decompress.com >b && sha1sum a b +/* +# data file is o/dbg/third_party/python/python.com +# level 0 147517 compress 495 MB/s decompress 1.4 GB/s +# level 1 80274 compress 29.2 MB/s decompress 303 MB/s +# level 2 79384 compress 33.8 MB/s decompress 212 MB/s +# level 3 78875 compress 28.9 MB/s decompress 224 MB/s +# level 4 78010 compress 27.1 MB/s decompress 319 MB/s <-- sweet spot? +# level 5 77107 compress 19.5 MB/s decompress 273 MB/s +# level 6 75081 compress 10.0 MB/s decompress 99.3 MB/s +# level 7 75022 compress 7.5 MB/s decompress 287 MB/s +# level 8 75016 compress 5.4 MB/s decompress 109 MB/s +# level 9 75016 compress 5.4 MB/s decompress 344 MB/s +m= +make -j8 MODE=$m o/$m/examples || exit +for level in $(seq 0 9); do + o/$m/examples/compress.com $level /tmp/info >/tmp/comp + compspeed=$(grep -Po '[.\d]+ \w+/s' /tmp/info) + o/$m/examples/decompress.com $level /tmp/info >/dev/null + decompspeed=$(grep -Po '[.\d]+ \w+/s' /tmp/info) + size=$(o/$m/examples/compress.com $level 1) { + level = atoi(argv[1]); + } else { + level = Z_DEFAULT_COMPRESSION; + } + rc = compressor(0, 1, level); if (rc == Z_OK) { return 0; } else { diff --git a/examples/decompress.c b/examples/decompress.c index a8b483b6d..657a0a4ed 100644 --- a/examples/decompress.c +++ b/examples/decompress.c @@ -14,10 +14,33 @@ #include "libc/stdio/stdio.h" #include "third_party/zlib/zlib.h" -#define CHUNK 4096 +#define CHUNK 32768 // clang-format off // make -j8 o//examples && dd if=/dev/urandom count=100 | tee a | o//examples/compress.com | o//examples/decompress.com >b && sha1sum a b +/* +# data file is o/dbg/third_party/python/python.com +# level 0 147517 compress 495 MB/s decompress 1.4 GB/s +# level 1 80274 compress 29.2 MB/s decompress 303 MB/s +# level 2 79384 compress 33.8 MB/s decompress 212 MB/s +# level 3 78875 compress 28.9 MB/s decompress 224 MB/s +# level 4 78010 compress 27.1 MB/s decompress 319 MB/s <-- sweet spot? +# level 5 77107 compress 19.5 MB/s decompress 273 MB/s +# level 6 75081 compress 10.0 MB/s decompress 99.3 MB/s +# level 7 75022 compress 7.5 MB/s decompress 287 MB/s +# level 8 75016 compress 5.4 MB/s decompress 109 MB/s +# level 9 75016 compress 5.4 MB/s decompress 344 MB/s +m= +make -j8 MODE=$m o/$m/examples || exit +for level in $(seq 0 9); do + o/$m/examples/compress.com $level /tmp/info >/tmp/comp + compspeed=$(grep -Po '[.\d]+ \w+/s' /tmp/info) + o/$m/examples/decompress.com $level /tmp/info >/dev/null + decompspeed=$(grep -Po '[.\d]+ \w+/s' /tmp/info) + size=$(o/$m/examples/compress.com $level : "); +} + int main(int argc, char *argv[]) { bool timeit; int64_t nanos; - int n, ws, pid; struct rusage ru; struct timespec ts1, ts2; char *prog, path[PATH_MAX]; sigset_t chldmask, savemask; - struct sigaction ignore, saveint, savequit; - char *p, *line, **args, *arg, *start, *state, prompt[64]; + int stdoutflags, stderrflags; + const char *stdoutpath, *stderrpath; + int n, rc, ws, pid, child, killcount; + struct sigaction sa, saveint, savequit; + char *p, *line, **args, *arg, *start, *state, prompt[1024]; linenoiseSetFreeHintsCallback(free); linenoiseSetHintsCallback(ShellHint); linenoiseSetCompletionCallback(ShellCompletion); - stpcpy(prompt, "$ "); + MakePrompt(prompt); while ((line = linenoiseWithHistory(prompt, "cmd"))) { n = 0; start = line; @@ -129,35 +157,114 @@ int main(int argc, char *argv[]) { } else { timeit = false; } + stdoutpath = 0; + stderrpath = 0; + stdoutflags = 0; + stderrflags = 0; args = xcalloc(1, sizeof(*args)); while ((arg = strtok_r(start, " \t\r\n", &state))) { - args = xrealloc(args, (++n + 1) * sizeof(*args)); - args[n - 1] = arg; - args[n - 0] = 0; - start = 0; + // cmd >>stdout.txt + if (arg[0] == '>' && arg[1] == '>') { + stdoutflags = O_WRONLY | O_APPEND | O_CREAT; + stdoutpath = arg + 2; + } else if (arg[0] == '>') { + // cmd >stdout.txt + stdoutflags = O_WRONLY | O_CREAT | O_TRUNC; + stdoutpath = arg + 1; + } else if (arg[0] == '2' && arg[1] == '>' && arg[2] == '>') { + // cmd 2>>stderr.txt + stderrflags = O_WRONLY | O_APPEND | O_CREAT; + stderrpath = arg + 3; + } else if (arg[0] == '2' && arg[1] == '>') { + // cmd 2>stderr.txt + stderrflags = O_WRONLY | O_CREAT | O_TRUNC; + stderrpath = arg + 2; + } else { + // arg + args = xrealloc(args, (++n + 1) * sizeof(*args)); + args[n - 1] = arg; + args[n - 0] = 0; + start = 0; + } } if (n > 0) { if ((prog = commandv(args[0], path, sizeof(path)))) { - ignore.sa_flags = 0; - ignore.sa_handler = SIG_IGN; - sigemptyset(&ignore.sa_mask); - sigaction(SIGINT, &ignore, &saveint); - sigaction(SIGQUIT, &ignore, &savequit); + + // let keyboard interrupts kill child and not shell + gotint = 0; + killcount = 0; + sa.sa_flags = 0; + sa.sa_handler = SIG_IGN; + sigemptyset(&sa.sa_mask); + sigaction(SIGQUIT, &sa, &savequit); + sa.sa_handler = OnInterrupt; + sigaction(SIGINT, &sa, &saveint); sigemptyset(&chldmask); sigaddset(&chldmask, SIGCHLD); sigprocmask(SIG_BLOCK, &chldmask, &savemask); + // record timestamp if (timeit) { clock_gettime(CLOCK_REALTIME, &ts1); } - if (!fork()) { + + // launch process + if (!(child = vfork())) { + if (stdoutpath) { + close(1); + open(stdoutpath, stdoutflags, 0644); + } + if (stderrpath) { + close(2); + open(stderrpath, stderrflags, 0644); + } sigaction(SIGINT, &saveint, 0); sigaction(SIGQUIT, &savequit, 0); sigprocmask(SIG_SETMASK, &savemask, 0); execv(prog, args); _Exit(127); } - wait4(0, &ws, 0, &ru); + + // wait for process + for (;;) { + if (gotint) { + switch (killcount) { + case 0: + // ctrl-c + // we do nothing + // terminals broadcast sigint to process group + rc = 0; + break; + case 1: + // ctrl-c ctrl-c + // we try sending sigterm + rc = kill(child, SIGTERM); + break; + default: + // ctrl-c ctrl-c ctrl-c ... + // we use kill -9 as our last resort + rc = kill(child, SIGKILL); + break; + } + if (rc == -1) { + fprintf(stderr, "kill failed: %m\n"); + exit(1); + } + ++killcount; + gotint = 0; + } + rc = wait4(0, &ws, 0, &ru); + if (rc != -1) { + break; + } else if (errno == EINTR) { + errno = 0; + } else { + fprintf(stderr, "wait failed: %m\n"); + exit(1); + } + } + + // print resource consumption for `time` pseudocommand if (timeit) { clock_gettime(CLOCK_REALTIME, &ts2); if (ts2.tv_sec == ts1.tv_sec) { @@ -174,23 +281,27 @@ int main(int argc, char *argv[]) { free(p); } + // update prompt to reflect exit status p = prompt; if (WIFEXITED(ws)) { if (WEXITSTATUS(ws)) { if (!__nocolor) p = stpcpy(p, "\e[1;31m"); p = stpcpy(p, "rc="); p = FormatInt32(p, WEXITSTATUS(ws)); + if (128 < WEXITSTATUS(ws) && WEXITSTATUS(ws) <= 128 + 32) { + *p++ = ' '; + p = stpcpy(p, strsignal(WEXITSTATUS(ws) - 128)); + } if (!__nocolor) p = stpcpy(p, "\e[0m"); *p++ = ' '; } } else { if (!__nocolor) p = stpcpy(p, "\e[1;31m"); - p = stpcpy(p, "rc="); p = stpcpy(p, strsignal(WTERMSIG(ws))); if (!__nocolor) p = stpcpy(p, "\e[0m"); *p++ = ' '; } - p = stpcpy(p, "$ "); + MakePrompt(p); sigaction(SIGINT, &saveint, 0); sigaction(SIGQUIT, &savequit, 0); diff --git a/libc/calls/kill-nt.c b/libc/calls/kill-nt.c index 9dc6d46a0..60f909fc3 100644 --- a/libc/calls/kill-nt.c +++ b/libc/calls/kill-nt.c @@ -20,18 +20,21 @@ #include "libc/calls/getconsolectrlevent.internal.h" #include "libc/calls/internal.h" #include "libc/dce.h" +#include "libc/intrin/kprintf.h" #include "libc/macros.internal.h" #include "libc/nt/console.h" #include "libc/nt/enum/ctrlevent.h" #include "libc/nt/enum/processaccess.h" +#include "libc/nt/enum/th32cs.h" #include "libc/nt/errors.h" #include "libc/nt/process.h" #include "libc/nt/runtime.h" +#include "libc/nt/struct/processentry32.h" #include "libc/sysv/errfuns.h" textwindows int sys_kill_nt(int pid, int sig) { bool32 ok; - int64_t handle; + int64_t h; int event, ntpid; // is killing everything except init really worth supporting? @@ -68,20 +71,37 @@ textwindows int sys_kill_nt(int pid, int sig) { } } - // XXX: Is this a cosmo pid that was returned by fork_nt? + // is this a cosmo pid that was returned by fork? if (__isfdkind(pid, kFdProcess)) { + // since windows can't execve we need to kill the grandchildren + // TODO(jart): should we just kill the whole tree too? there's + // no obvious way to tell if it's the execve shell + int64_t hSnap, hProc, hChildProc; + struct NtProcessEntry32 pe = {.dwSize = sizeof(struct NtProcessEntry32)}; + ntpid = GetProcessId(g_fds.p[pid].handle); + hSnap = CreateToolhelp32Snapshot(kNtTh32csSnapprocess, 0); + if (Process32First(hSnap, &pe)) { + do { + if (pe.th32ParentProcessID == ntpid) { + if ((h = OpenProcess(kNtProcessTerminate, false, pe.th32ProcessID))) { + TerminateProcess(h, 128 + sig); + CloseHandle(h); + } + } + } while (Process32Next(hSnap, &pe)); + } ok = TerminateProcess(g_fds.p[pid].handle, 128 + sig); if (!ok && GetLastError() == kNtErrorAccessDenied) ok = true; return 0; } // XXX: Is this a raw new technology pid? Because that's messy. - if ((handle = OpenProcess(kNtProcessTerminate, false, pid))) { - ok = TerminateProcess(handle, 128 + sig); + if ((h = OpenProcess(kNtProcessTerminate, false, pid))) { + ok = TerminateProcess(h, 128 + sig); if (!ok && GetLastError() == kNtErrorAccessDenied) { ok = true; // cargo culting other codebases here } - CloseHandle(handle); + CloseHandle(h); return 0; } else { return -1; diff --git a/libc/calls/sigaltstack.c b/libc/calls/sigaltstack.c index 4ec6f30f5..4839b9dc3 100644 --- a/libc/calls/sigaltstack.c +++ b/libc/calls/sigaltstack.c @@ -57,8 +57,9 @@ static void sigaltstack2linux(struct sigaltstack *linux, * struct sigaction sa; * struct sigaltstack ss; * ss.ss_flags = 0; - * ss.ss_size = SIGSTKSZ; - * ss.ss_sp = malloc(ss.ss_size); + * ss.ss_size = GetStackSize(); + * ss.ss_sp = mmap(0, GetStackSize(), PROT_READ | PROT_WRITE, + * MAP_STACK | MAP_ANONYMOUS, -1, 0); * sa.sa_flags = SA_ONSTACK; * sa.sa_handler = OnStackOverflow; * __cxa_atexit(free, ss[0].ss_sp, 0); @@ -66,6 +67,11 @@ static void sigaltstack2linux(struct sigaltstack *linux, * sigaltstack(&ss, 0); * sigaction(SIGSEGV, &sa, 0); * + * It's strongly recommended that you allocate a stack with the same + * size as GetStackSize() and that it have GetStackSize() alignment. + * Otherwise some of your runtime support code (e.g. ftrace stack use + * logging, kprintf() memory safety) won't be able to work as well. + * * @param neu if non-null will install new signal alt stack * @param old if non-null will receive current signal alt stack * @return 0 on success, or -1 w/ errno diff --git a/libc/calls/wincrash.c b/libc/calls/wincrash.c index fbe717a5e..7c850d6ec 100644 --- a/libc/calls/wincrash.c +++ b/libc/calls/wincrash.c @@ -35,6 +35,7 @@ privileged unsigned __wincrash(struct NtExceptionPointers *ep) { int sig, code; ucontext_t ctx; STRACE("__wincrash"); + switch (ep->ExceptionRecord->ExceptionCode) { case kNtSignalBreakpoint: code = TRAP_BRKPT; diff --git a/libc/fmt/fmt.mk b/libc/fmt/fmt.mk index ffd33e912..6ff36f863 100644 --- a/libc/fmt/fmt.mk +++ b/libc/fmt/fmt.mk @@ -77,6 +77,16 @@ o/$(MODE)/libc/fmt/wcstoumax.o: \ OVERRIDE_CFLAGS += \ -Os +# we can't use compiler magic because: +# kprintf() depends on these functions +o/$(MODE)/libc/fmt/strerrno.greg.o \ +o/$(MODE)/libc/fmt/strerrdoc.greg.o \ +o/$(MODE)/libc/fmt/strerror_wr.greg.o: \ + OVERRIDE_CFLAGS += \ + -fpie \ + -ffreestanding \ + $(NO_MAGIC) + LIBC_FMT_LIBS = $(foreach x,$(LIBC_FMT_ARTIFACTS),$($(x))) LIBC_FMT_SRCS = $(foreach x,$(LIBC_FMT_ARTIFACTS),$($(x)_SRCS)) LIBC_FMT_HDRS = $(foreach x,$(LIBC_FMT_ARTIFACTS),$($(x)_HDRS)) diff --git a/libc/fmt/strerror_r.greg.c b/libc/fmt/strerror_r.c similarity index 100% rename from libc/fmt/strerror_r.greg.c rename to libc/fmt/strerror_r.c diff --git a/libc/integral/c.inc b/libc/integral/c.inc index 57ea8b3d6..05234e6d8 100644 --- a/libc/integral/c.inc +++ b/libc/integral/c.inc @@ -195,9 +195,9 @@ typedef struct { #ifndef privileged #if !defined(__STRICT_ANSI__) && \ (__has_attribute(__visibility__) || defined(__GNUC__)) -#define privileged _Section(".privileged") noinstrument +#define privileged _Section(".privileged") #else -#define privileged _Section(".privileged") noinstrument +#define privileged _Section(".privileged") #endif #endif diff --git a/libc/intrin/_spinlock_debug_1.c b/libc/intrin/_spinlock_debug_1.c new file mode 100644 index 000000000..126e47099 --- /dev/null +++ b/libc/intrin/_spinlock_debug_1.c @@ -0,0 +1,56 @@ +/*-*- mode:c;indent-tabs-mode:nil;c-basic-offset:2;tab-width:8;coding:utf-8 -*-│ +│vi: set net ft=c ts=2 sts=2 sw=2 fenc=utf-8 :vi│ +╞══════════════════════════════════════════════════════════════════════════════╡ +│ Copyright 2022 Justine Alexandra Roberts Tunney │ +│ │ +│ Permission to use, copy, modify, and/or distribute this software for │ +│ any purpose with or without fee is hereby granted, provided that the │ +│ above copyright notice and this permission notice appear in all copies. │ +│ │ +│ THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL │ +│ WARRANTIES WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED │ +│ WARRANTIES OF MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE │ +│ AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL │ +│ DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR │ +│ PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER │ +│ TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR │ +│ PERFORMANCE OF THIS SOFTWARE. │ +╚─────────────────────────────────────────────────────────────────────────────*/ +#include "libc/calls/calls.h" +#include "libc/intrin/kprintf.h" +#include "libc/intrin/lockcmpxchgp.h" +#include "libc/intrin/spinlock.h" +#include "libc/nexgen32e/rdtsc.h" +#include "libc/time/clockstonanos.internal.h" + +void _spinlock_debug_1(void *lockptr, const char *lockname, const char *file, + int line, const char *func) { + unsigned i; + uint64_t ts1, ts2; + int me, owner, *lock = lockptr; + me = gettid(); + owner = 0; + if (!_lockcmpxchgp(lock, &owner, me)) { + if (owner == me) { + kprintf("%s:%d: warning: possible re-entry on lock %s in %s()\n", file, + line, lockname, func); + } + i = 0; + ts1 = rdtsc(); + for (;;) { + owner = 0; + if (_lockcmpxchgp(lock, &owner, me)) break; + ts2 = rdtsc(); + if (ClocksToNanos(ts1, ts2) > 1000000000ul) { + ts1 = ts2; + kprintf("%s:%d: warning: slow lock on %s in %s()\n", file, line, + lockname, func); + } + if (++i & 7) { + __builtin_ia32_pause(); + } else { + sched_yield(); + } + } + } +} diff --git a/libc/intrin/_spinlock_debug_4.c b/libc/intrin/_spinlock_debug_4.c new file mode 100644 index 000000000..e42dfbbcf --- /dev/null +++ b/libc/intrin/_spinlock_debug_4.c @@ -0,0 +1,57 @@ +/*-*- mode:c;indent-tabs-mode:nil;c-basic-offset:2;tab-width:8;coding:utf-8 -*-│ +│vi: set net ft=c ts=2 sts=2 sw=2 fenc=utf-8 :vi│ +╞══════════════════════════════════════════════════════════════════════════════╡ +│ Copyright 2022 Justine Alexandra Roberts Tunney │ +│ │ +│ Permission to use, copy, modify, and/or distribute this software for │ +│ any purpose with or without fee is hereby granted, provided that the │ +│ above copyright notice and this permission notice appear in all copies. │ +│ │ +│ THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL │ +│ WARRANTIES WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED │ +│ WARRANTIES OF MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE │ +│ AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL │ +│ DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR │ +│ PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER │ +│ TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR │ +│ PERFORMANCE OF THIS SOFTWARE. │ +╚─────────────────────────────────────────────────────────────────────────────*/ +#include "libc/calls/calls.h" +#include "libc/intrin/kprintf.h" +#include "libc/intrin/lockcmpxchgp.h" +#include "libc/intrin/spinlock.h" +#include "libc/nexgen32e/rdtsc.h" +#include "libc/time/clockstonanos.internal.h" + +void _spinlock_debug_4(void *lockptr, const char *lockname, const char *file, + int line, const char *func) { + unsigned i; + uint64_t ts1, ts2; + int me, owner, *lock = lockptr; + me = gettid(); + owner = 0; + if (!_lockcmpxchgp(lock, &owner, me)) { + if (owner == me) { + kprintf("%s:%d: warning: lock re-entry on %s in %s()\n", file, line, + lockname, func); + _Exit(1); + } + i = 0; + ts1 = rdtsc(); + for (;;) { + owner = 0; + if (_lockcmpxchgp(lock, &owner, me)) break; + ts2 = rdtsc(); + if (ClocksToNanos(ts1, ts2) > 1000000000ul) { + ts1 = ts2; + kprintf("%s:%d: warning: slow lock on %s in %s()\n", file, line, + lockname, func); + } + if (++i & 7) { + __builtin_ia32_pause(); + } else { + sched_yield(); + } + } + } +} diff --git a/libc/intrin/asan.c b/libc/intrin/asan.c index 7b17daa53..d8bf1bd89 100644 --- a/libc/intrin/asan.c +++ b/libc/intrin/asan.c @@ -59,9 +59,21 @@ STATIC_YOINK("_init_asan"); +#if IsModeDbg() +// MODE=dbg +// O(32mb) of morgue memory +// Θ(64) bytes of malloc overhead #define ASAN_MORGUE_ITEMS 512 -#define ASAN_MORGUE_THRESHOLD 65536 // morgue memory O(ITEMS*THRESHOLD) -#define ASAN_TRACE_ITEMS 16 // backtrace limit on malloc origin +#define ASAN_MORGUE_THRESHOLD 65536 +#define ASAN_TRACE_ITEMS 16 +#else +// MODE=asan +// O(32mb) of morgue memory +// Θ(32) bytes of malloc overhead +#define ASAN_MORGUE_ITEMS 512 +#define ASAN_MORGUE_THRESHOLD 65536 +#define ASAN_TRACE_ITEMS 4 +#endif /** * @fileoverview Cosmopolitan Address Sanitizer Runtime. @@ -853,7 +865,7 @@ static void __asan_morgue_flush(void) { void *p; _spinlock_cooperative(&__asan_lock); for (i = 0; i < ARRAYLEN(__asan_morgue.p); ++i) { - if (weaken(dlfree)) { + if (__asan_morgue.p[i] && weaken(dlfree)) { weaken(dlfree)(__asan_morgue.p[i]); } __asan_morgue.p[i] = 0; @@ -1196,9 +1208,9 @@ void __asan_evil(uint8_t *addr, int size, const char *s1, const char *s2) { struct AsanTrace tr; __asan_rawtrace(&tr, __builtin_frame_address(0)); kprintf( - "WARNING: ASAN %s %s bad %d byte %s at %x bt %x %x %x %x %x\n", + "WARNING: ASAN %s %s bad %d byte %s at %x bt %x %x %x\n", __asan_noreentry == gettid() ? "error during" : "multi-threaded crash", - s1, size, s2, addr, tr.p[0], tr.p[1], tr.p[2], tr.p[3], tr.p[4], tr.p[5]); + s1, size, s2, addr, tr.p[0], tr.p[1], tr.p[2], tr.p[3]); } void __asan_report_load(uint8_t *addr, int size) { diff --git a/libc/intrin/getmagnumstr.greg.c b/libc/intrin/getmagnumstr.greg.c index 20834e936..7bcbbc542 100644 --- a/libc/intrin/getmagnumstr.greg.c +++ b/libc/intrin/getmagnumstr.greg.c @@ -18,7 +18,7 @@ ╚─────────────────────────────────────────────────────────────────────────────*/ #include "libc/fmt/magnumstrs.internal.h" -char *GetMagnumStr(const struct MagnumStr *ms, int x) { +privileged char *GetMagnumStr(const struct MagnumStr *ms, int x) { int i; for (i = 0; ms[i].x != MAGNUM_TERMINATOR; ++i) { if (x == MAGNUM_NUMBER(ms, i)) { diff --git a/libc/intrin/gettid.greg.c b/libc/intrin/gettid.greg.c index 4331cfbd8..95a77a7ad 100644 --- a/libc/intrin/gettid.greg.c +++ b/libc/intrin/gettid.greg.c @@ -30,6 +30,13 @@ privileged int gettid(void) { int64_t wut; struct WinThread *wt; + if (__tls_enabled) { + rc = *(int *)(__get_tls() + 0x38); + if (rc && rc != -1) { + return rc; + } + } + if (IsWindows()) { return GetCurrentThreadId(); } diff --git a/libc/intrin/intrin.mk b/libc/intrin/intrin.mk index f3f671ea5..7e10b36e9 100644 --- a/libc/intrin/intrin.mk +++ b/libc/intrin/intrin.mk @@ -58,11 +58,21 @@ o/$(MODE)/libc/intrin/asan.o: \ -finline \ -finline-functions +# we can't use compiler magic because: +# kprintf() is mission critical to error reporting +o/$(MODE)/libc/intrin/getmagnumstr.greg.o \ +o/$(MODE)/libc/intrin/strerrno.greg.o \ +o/$(MODE)/libc/intrin/strerrdoc.greg.o \ +o/$(MODE)/libc/intrin/strerror_wr.greg.o \ o/$(MODE)/libc/intrin/kprintf.greg.o: \ OVERRIDE_CFLAGS += \ -fpie \ + -fwrapv \ + -x-no-pg \ + -mno-fentry \ -ffreestanding \ - $(NO_MAGIC) + -fno-sanitize=all \ + -fno-stack-protector o/$(MODE)/libc/intrin/tls.greg.o \ o/$(MODE)/libc/intrin/exit.greg.o \ diff --git a/libc/fmt/kerrnodocs.S b/libc/intrin/kerrnodocs.S similarity index 100% rename from libc/fmt/kerrnodocs.S rename to libc/intrin/kerrnodocs.S diff --git a/libc/fmt/kerrnonames.S b/libc/intrin/kerrnonames.S similarity index 100% rename from libc/fmt/kerrnonames.S rename to libc/intrin/kerrnonames.S diff --git a/libc/intrin/kprintf.greg.c b/libc/intrin/kprintf.greg.c index c07ecd2e8..034075d47 100644 --- a/libc/intrin/kprintf.greg.c +++ b/libc/intrin/kprintf.greg.c @@ -52,6 +52,8 @@ #include "libc/sysv/consts/prot.h" #include "libc/time/clockstonanos.internal.h" +extern hidden struct SymbolTable *__symtab; + struct Timestamps { unsigned long long birth; unsigned long long start; @@ -515,13 +517,19 @@ privileged static size_t kformat(char *b, size_t n, const char *fmt, va_list va, } case 't': { + // %t will print the &symbol associated with an address. this + // requires that some other code linked GetSymbolTable() and + // called it beforehand to ensure the symbol table is loaded. + // if the symbol table isn't linked or available, then this + // routine will display &hexaddr so objdump -dS foo.com.dbg + // can be manually consulted to look up the faulting code. int idx; x = va_arg(va, intptr_t); - if (weaken(__get_symbol) && + if (weaken(__symtab) && *weaken(__symtab) && (idx = weaken(__get_symbol)(0, x)) != -1) { if (p + 1 <= e) *p++ = '&'; - s = weaken(GetSymbolTable)()->name_base + - weaken(GetSymbolTable)()->names[idx]; + s = (*weaken(__symtab))->name_base + + (*weaken(__symtab))->names[idx]; goto FormatString; } base = 4; diff --git a/libc/intrin/lockcmpxchgp.h b/libc/intrin/lockcmpxchgp.h new file mode 100644 index 000000000..fdf12d3ab --- /dev/null +++ b/libc/intrin/lockcmpxchgp.h @@ -0,0 +1,24 @@ +#ifndef COSMOPOLITAN_LIBC_INTRIN_LOCKCMPXCHGP_H_ +#define COSMOPOLITAN_LIBC_INTRIN_LOCKCMPXCHGP_H_ +#include "libc/bits/asmflag.h" +#if !(__ASSEMBLER__ + __LINKER__ + 0) +COSMOPOLITAN_C_START_ + +#if defined(__GNUC__) && !defined(__STRICT_ANSI__) && defined(__x86__) +#define _lockcmpxchgp(IN_OUT_IFTHING, IN_OUT_ISEQUALTOME, IN_REPLACEITWITHME) \ + ({ \ + bool DidIt; \ + autotype(IN_OUT_IFTHING) IfThing = (IN_OUT_IFTHING); \ + typeof(IfThing) IsEqualToMe = (IN_OUT_ISEQUALTOME); \ + typeof(*IfThing) ReplaceItWithMe = (IN_REPLACEITWITHME); \ + asm volatile(ZFLAG_ASM("lock cmpxchg\t%3,%1") \ + : ZFLAG_CONSTRAINT(DidIt), "+m"(*IfThing), "+a"(*IsEqualToMe) \ + : "r"(ReplaceItWithMe) \ + : "cc"); \ + DidIt; \ + }) +#endif /* GNUC && !ANSI && x86 */ + +COSMOPOLITAN_C_END_ +#endif /* !(__ASSEMBLER__ + __LINKER__ + 0) */ +#endif /* COSMOPOLITAN_LIBC_INTRIN_LOCKCMPXCHGP_H_ */ diff --git a/libc/intrin/spinlock.h b/libc/intrin/spinlock.h index c58c72f23..0b6d9eb8c 100644 --- a/libc/intrin/spinlock.h +++ b/libc/intrin/spinlock.h @@ -1,8 +1,9 @@ #ifndef COSMOPOLITAN_LIBC_INTRIN_SPINLOCK_H_ #define COSMOPOLITAN_LIBC_INTRIN_SPINLOCK_H_ +#include "libc/assert.h" #include "libc/calls/calls.h" -#include "libc/intrin/kprintf.h" #include "libc/intrin/lockcmpxchg.h" +#include "libc/intrin/lockcmpxchgp.h" #if IsModeDbg() && !defined(_SPINLOCK_DEBUG) #define _SPINLOCK_DEBUG @@ -19,63 +20,62 @@ #define _spinlock_ndebug(lock) _spinlock_cooperative(lock) #endif -#define _spunlock(lock) __atomic_clear(lock, __ATOMIC_RELAXED) - #define _trylock(lock) __atomic_test_and_set(lock, __ATOMIC_SEQ_CST) -#define _seizelock(lock) \ - do { \ - typeof(*(lock)) x = 1; \ - __atomic_store(lock, &x, __ATOMIC_RELEASE); \ +#define _spunlock(lock) \ + do { \ + autotype(lock) __lock = (lock); \ + typeof(*__lock) __x = 0; \ + __atomic_store(__lock, &__x, __ATOMIC_RELAXED); \ } while (0) -#define _spinlock_tiny(lock) \ - do { \ - while (_trylock(lock)) { \ - __builtin_ia32_pause(); \ - } \ +#define _seizelock(lock) \ + do { \ + autotype(lock) __lock = (lock); \ + typeof(*__lock) __x = 1; \ + __atomic_store(__lock, &__x, __ATOMIC_RELEASE); \ } while (0) -#define _spinlock_cooperative(lock) \ - do { \ - int __tries = 0; \ - for (;;) { \ - typeof(*(lock)) x; \ - __atomic_load(lock, &x, __ATOMIC_RELAXED); \ - if (!x && !_trylock(lock)) { \ - break; \ - } else if (++__tries & 7) { \ - __builtin_ia32_pause(); \ - } else { \ - sched_yield(); \ - } \ - } \ +#define _spinlock_tiny(lock) \ + do { \ + autotype(lock) __lock = (lock); \ + while (_trylock(__lock)) { \ + __builtin_ia32_pause(); \ + } \ } while (0) -#define _spinlock_debug(lock) \ - do { \ - typeof(*(lock)) me, owner; \ - unsigned long warntries = 16777216; \ - me = gettid(); \ - if (!_lockcmpxchg(lock, 0, me)) { \ - __atomic_load(lock, &owner, __ATOMIC_RELAXED); \ - if (owner == me) { \ - kprintf("%s:%d: warning: possible re-entry on %s in %s()\n", __FILE__, \ - __LINE__, #lock, __FUNCTION__); \ - } \ - while (!_lockcmpxchg(lock, 0, me)) { \ - if (!--warntries) { \ - warntries = -1; \ - kprintf("%s:%d: warning: possible deadlock on %s in %s()\n", \ - __FILE__, __LINE__, #lock, __FUNCTION__); \ - } \ - if (warntries & 7) { \ - __builtin_ia32_pause(); \ - } else { \ - sched_yield(); \ - } \ - } \ - } \ +#define _spinlock_cooperative(lock) \ + do { \ + autotype(lock) __lock = (lock); \ + typeof(*__lock) __x; \ + int __tries = 0; \ + for (;;) { \ + __atomic_load(__lock, &__x, __ATOMIC_RELAXED); \ + if (!__x && !_trylock(__lock)) { \ + break; \ + } else if (++__tries & 7) { \ + __builtin_ia32_pause(); \ + } else { \ + sched_yield(); \ + } \ + } \ + } while (0) + +void _spinlock_debug_1(void *, const char *, const char *, int, const char *); +void _spinlock_debug_4(void *, const char *, const char *, int, const char *); + +#define _spinlock_debug(lock) \ + do { \ + switch (sizeof(*(lock))) { \ + case 1: \ + _spinlock_debug_1(lock, #lock, __FILE__, __LINE__, __FUNCTION__); \ + break; \ + case 4: \ + _spinlock_debug_4(lock, #lock, __FILE__, __LINE__, __FUNCTION__); \ + break; \ + default: \ + assert(!"unsupported size"); \ + } \ } while (0) #endif /* COSMOPOLITAN_LIBC_INTRIN_SPINLOCK_H_ */ diff --git a/libc/fmt/strerdoc.greg.c b/libc/intrin/strerdoc.greg.c similarity index 98% rename from libc/fmt/strerdoc.greg.c rename to libc/intrin/strerdoc.greg.c index b8e89b384..07022eca7 100644 --- a/libc/fmt/strerdoc.greg.c +++ b/libc/intrin/strerdoc.greg.c @@ -23,7 +23,7 @@ * Converts errno value to descriptive sentence. * @return non-null rodata string or null if not found */ -char *strerdoc(int x) { +privileged char *strerdoc(int x) { if (x) { return GetMagnumStr(kErrnoDocs, x); } else { diff --git a/libc/fmt/strerrno.greg.c b/libc/intrin/strerrno.greg.c similarity index 98% rename from libc/fmt/strerrno.greg.c rename to libc/intrin/strerrno.greg.c index 145727831..3c9e1a037 100644 --- a/libc/fmt/strerrno.greg.c +++ b/libc/intrin/strerrno.greg.c @@ -23,7 +23,7 @@ * Converts errno value to symbolic name. * @return non-null rodata string or null if not found */ -char *strerrno(int x) { +privileged char *strerrno(int x) { if (x) { return GetMagnumStr(kErrnoNames, x); } else { diff --git a/libc/fmt/strerror_wr.greg.c b/libc/intrin/strerror_wr.greg.c similarity index 97% rename from libc/fmt/strerror_wr.greg.c rename to libc/intrin/strerror_wr.greg.c index 8fad73863..7832eb360 100644 --- a/libc/fmt/strerror_wr.greg.c +++ b/libc/intrin/strerror_wr.greg.c @@ -31,7 +31,7 @@ * @param err is error number or zero if unknown * @return 0 on success, or error code */ -int strerror_wr(int err, uint32_t winerr, char *buf, size_t size) { +privileged int strerror_wr(int err, uint32_t winerr, char *buf, size_t size) { /* kprintf() weakly depends on this function */ int c, n; char16_t winmsg[256]; diff --git a/libc/intrin/tls.greg.c b/libc/intrin/tls.greg.c index 358172228..52366662c 100644 --- a/libc/intrin/tls.greg.c +++ b/libc/intrin/tls.greg.c @@ -32,10 +32,24 @@ /** * Initializes thread information block. + * + * Here's the layout your c library assumes: + * + * offset size description + * 0x0000 0x08 linear address pointer + * 0x0008 0x08 jmp_buf *exiter + * 0x0010 0x04 exit code + * 0x0030 0x08 linear address pointer + * 0x0038 0x04 tid + * 0x003c 0x04 errno + * */ privileged void *__initialize_tls(char tib[hasatleast 64]) { *(intptr_t *)tib = (intptr_t)tib; + *(intptr_t *)(tib + 0x08) = 0; + *(int *)(tib + 0x10) = -1; // exit code *(intptr_t *)(tib + 0x30) = (intptr_t)tib; + *(int *)(tib + 0x38) = -1; // tid *(int *)(tib + 0x3c) = __errno; return tib; } diff --git a/libc/log/checkfail.c b/libc/log/checkfail.c index ba43b5123..b6efa27a1 100644 --- a/libc/log/checkfail.c +++ b/libc/log/checkfail.c @@ -68,7 +68,7 @@ relegated void __check_fail(const char *suffix, const char *opstr, } kprintf("%s\n", RESET); if (!IsTiny() && e == ENOMEM) { - PrintMemoryIntervals(2, &_mmi); + __print_maps(); } __die(); unreachable; diff --git a/libc/log/internal.h b/libc/log/internal.h index 14a644ca8..625da7514 100644 --- a/libc/log/internal.h +++ b/libc/log/internal.h @@ -10,11 +10,11 @@ COSMOPOLITAN_C_START_ extern hidden bool __nocolor; extern hidden int kCrashSigs[7]; extern hidden bool g_isrunningundermake; -extern hidden struct sigaction g_oldcrashacts[7]; void __start_fatal(const char *, int) hidden; void __oncrash(int, struct siginfo *, struct ucontext *) relegated; void __restore_tty(void); +void RestoreDefaultCrashSignalHandlers(void); COSMOPOLITAN_C_END_ #endif /* !(__ASSEMBLER__ + __LINKER__ + 0) */ diff --git a/libc/log/leaks.c b/libc/log/leaks.c index 0f61dda6a..16c65ceee 100644 --- a/libc/log/leaks.c +++ b/libc/log/leaks.c @@ -108,7 +108,7 @@ noasan void CheckForMemoryLeaks(void) { } malloc_inspect_all(OnMemory, 0); kprintf("\n"); - PrintMemoryIntervals(2, &_mmi); + __print_maps(); /* PrintSystemMappings(2); */ /* PrintGarbage(); */ __restorewintty(); diff --git a/libc/log/oncrash.c b/libc/log/oncrash.c index 325d0273a..35b26d413 100644 --- a/libc/log/oncrash.c +++ b/libc/log/oncrash.c @@ -35,6 +35,7 @@ #include "libc/nexgen32e/stackframe.h" #include "libc/runtime/internal.h" #include "libc/runtime/pc.internal.h" +#include "libc/runtime/runtime.h" /** * @fileoverview Abnormal termination handling & GUI debugging. @@ -57,7 +58,6 @@ static const char kCpuFlags[12] forcealign(1) = "CVPRAKZSTIDO"; static const char kFpuExceptions[6] forcealign(1) = "IDZOUP"; int kCrashSigs[7]; -struct sigaction g_oldcrashacts[7]; relegated static void ShowFunctionCalls(ucontext_t *ctx) { struct StackFrame *bp; @@ -220,7 +220,7 @@ relegated void ShowCrashReport(int err, int sig, struct siginfo *si, ShowSseRegisters(ctx); } kprintf("\n"); - PrintMemoryIntervals(2, &_mmi); + __print_maps(); /* PrintSystemMappings(2); */ if (__argv) { for (i = 0; i < __argc; ++i) { @@ -232,16 +232,6 @@ relegated void ShowCrashReport(int err, int sig, struct siginfo *si, kprintf("\n"); } -relegated static void RestoreDefaultCrashSignalHandlers(void) { - size_t i; - sigset_t ss; - sigemptyset(&ss); - sigprocmask(SIG_SETMASK, &ss, NULL); - for (i = 0; i < ARRAYLEN(kCrashSigs); ++i) { - if (kCrashSigs[i]) sigaction(kCrashSigs[i], &g_oldcrashacts[i], NULL); - } -} - static wontreturn relegated noinstrument void __minicrash(int sig, struct siginfo *si, ucontext_t *ctx, diff --git a/libc/log/showcrashreports.c b/libc/log/showcrashreports.c index 86efb79ad..29bdfeddf 100644 --- a/libc/log/showcrashreports.c +++ b/libc/log/showcrashreports.c @@ -16,13 +16,17 @@ │ TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR │ │ PERFORMANCE OF THIS SOFTWARE. │ ╚─────────────────────────────────────────────────────────────────────────────*/ +#include "libc/calls/calls.h" #include "libc/calls/sigbits.h" #include "libc/calls/struct/sigaction.h" #include "libc/calls/struct/sigaltstack.h" #include "libc/log/internal.h" #include "libc/log/log.h" #include "libc/macros.internal.h" +#include "libc/runtime/stack.h" #include "libc/runtime/symbols.internal.h" +#include "libc/sysv/consts/map.h" +#include "libc/sysv/consts/prot.h" #include "libc/sysv/consts/sa.h" #include "libc/sysv/consts/sig.h" #include "libc/sysv/consts/ss.h" @@ -31,12 +35,43 @@ STATIC_YOINK("__die"); /* for backtracing */ STATIC_YOINK("malloc_inspect_all"); /* for asan memory origin */ STATIC_YOINK("__get_symbol_by_addr"); /* for asan memory origin */ -static struct sigaltstack oldsigaltstack; extern const unsigned char __oncrash_thunks[8][11]; +static struct sigaltstack g_oldsigaltstack; +static struct sigaction g_oldcrashacts[7]; + +static void InstallCrashHandlers(int extraflags) { + size_t i; + struct sigaction sa; + bzero(&sa, sizeof(sa)); + sa.sa_flags = SA_SIGINFO | SA_NODEFER | extraflags; + sigfillset(&sa.sa_mask); + for (i = 0; i < ARRAYLEN(kCrashSigs); ++i) { + sigdelset(&sa.sa_mask, kCrashSigs[i]); + } + for (i = 0; i < ARRAYLEN(kCrashSigs); ++i) { + if (kCrashSigs[i]) { + sa.sa_sigaction = (sigaction_f)__oncrash_thunks[i]; + sigaction(kCrashSigs[i], &sa, &g_oldcrashacts[i]); + } + } +} + +relegated void RestoreDefaultCrashSignalHandlers(void) { + size_t i; + sigset_t ss; + sigemptyset(&ss); + sigprocmask(SIG_SETMASK, &ss, NULL); + for (i = 0; i < ARRAYLEN(kCrashSigs); ++i) { + if (kCrashSigs[i]) { + sigaction(kCrashSigs[i], &g_oldcrashacts[i], NULL); + } + } +} static void FreeSigAltStack(void *p) { - sigaltstack(&oldsigaltstack, 0); - free(p); + InstallCrashHandlers(0); + sigaltstack(&g_oldsigaltstack, 0); + munmap(p, GetStackSize()); } /** @@ -57,8 +92,6 @@ static void FreeSigAltStack(void *p) { * @see callexitontermination() */ void ShowCrashReports(void) { - size_t i; - struct sigaction sa; struct sigaltstack ss; /* : showcrashreports.c, oncrashthunks.S, oncrash.c */ kCrashSigs[0] = SIGQUIT; /* ctrl+\ aka ctrl+break */ @@ -73,25 +106,17 @@ void ShowCrashReports(void) { bzero(&ss, sizeof(ss)); ss.ss_flags = 0; ss.ss_size = SIGSTKSZ; - if ((ss.ss_sp = malloc(SIGSTKSZ))) { - if (!sigaltstack(&ss, &oldsigaltstack)) { + if ((ss.ss_sp = mmap(0, GetStackSize(), PROT_READ | PROT_WRITE, + MAP_STACK | MAP_ANONYMOUS, -1, 0))) { + if (!sigaltstack(&ss, &g_oldsigaltstack)) { __cxa_atexit(FreeSigAltStack, ss.ss_sp, 0); } else { - free(ss.ss_sp); + munmap(ss.ss_sp, GetStackSize()); } } - } - bzero(&sa, sizeof(sa)); - sa.sa_flags = SA_SIGINFO | SA_NODEFER | SA_ONSTACK; - sigfillset(&sa.sa_mask); - for (i = 0; i < ARRAYLEN(kCrashSigs); ++i) { - sigdelset(&sa.sa_mask, kCrashSigs[i]); - } - for (i = 0; i < ARRAYLEN(kCrashSigs); ++i) { - if (kCrashSigs[i]) { - sa.sa_sigaction = (sigaction_f)__oncrash_thunks[i]; - sigaction(kCrashSigs[i], &sa, &g_oldcrashacts[i]); - } + InstallCrashHandlers(SA_ONSTACK); + } else { + InstallCrashHandlers(0); } GetSymbolTable(); } diff --git a/libc/nexgen32e/checkstackalign.S b/libc/nexgen32e/checkstackalign.S index e362d7854..e944f1b38 100644 --- a/libc/nexgen32e/checkstackalign.S +++ b/libc/nexgen32e/checkstackalign.S @@ -21,7 +21,7 @@ // Checks that stack is 16-byte aligned. // // This function crashes if called with a misaligned stack. -_checkstackalign: +CheckStackIsAligned: push %rbp mov %rsp,%rbp @@ -35,4 +35,4 @@ _checkstackalign: leave ret - .endfn _checkstackalign,globl + .endfn CheckStackIsAligned,globl diff --git a/libc/nexgen32e/nexgen32e.h b/libc/nexgen32e/nexgen32e.h index e9f7dacf0..c2d45ee3f 100644 --- a/libc/nexgen32e/nexgen32e.h +++ b/libc/nexgen32e/nexgen32e.h @@ -7,7 +7,7 @@ extern long kHalfCache3; void imapxlatab(void *); void insertionsort(int32_t *, size_t); -void _checkstackalign(void); +void CheckStackIsAligned(void); int64_t div10int64(int64_t) libcesque pureconst; int64_t div100int64(int64_t) libcesque pureconst; diff --git a/libc/nexgen32e/nt2sysv.S b/libc/nexgen32e/nt2sysv.S index ef5939d5b..34ae867d9 100644 --- a/libc/nexgen32e/nt2sysv.S +++ b/libc/nexgen32e/nt2sysv.S @@ -32,6 +32,8 @@ __nt2sysv: push %rbp mov %rsp,%rbp +// TODO(jart): We should probably find some way to use our own +// stack when Windows delivers signals ;_; .profilable sub $0x100,%rsp push %rbx diff --git a/libc/nt/enum/th32cs.h b/libc/nt/enum/th32cs.h new file mode 100644 index 000000000..9c870ecf0 --- /dev/null +++ b/libc/nt/enum/th32cs.h @@ -0,0 +1,11 @@ +#ifndef COSMOPOLITAN_LIBC_NT_ENUM_TH32CS_H_ +#define COSMOPOLITAN_LIBC_NT_ENUM_TH32CS_H_ + +#define kNtTh32csInherit 0x80000000 +#define kNtTh32csSnapheaplist 0x00000001 +#define kNtTh32csSnapmodule 0x00000008 +#define kNtTh32csSnapmodule32 0x00000010 +#define kNtTh32csSnapprocess 0x00000002 +#define kNtTh32csSnapthread 0x00000004 + +#endif /* COSMOPOLITAN_LIBC_NT_ENUM_TH32CS_H_ */ diff --git a/libc/nt/kernel32/CreateToolhelp32Snapshot.s b/libc/nt/kernel32/CreateToolhelp32Snapshot.s index 54982f143..aea3d4177 100644 --- a/libc/nt/kernel32/CreateToolhelp32Snapshot.s +++ b/libc/nt/kernel32/CreateToolhelp32Snapshot.s @@ -1,2 +1,12 @@ .include "o/libc/nt/codegen.inc" -.imp kernel32,__imp_CreateToolhelp32Snapshot,CreateToolhelp32Snapshot,250 +.imp kernel32,__imp_CreateToolhelp32Snapshot,CreateToolhelp32Snapshot,0 + + .text.windows +CreateToolhelp32Snapshot: + push %rbp + mov %rsp,%rbp + .profilable + mov __imp_CreateToolhelp32Snapshot(%rip),%rax + jmp __sysv2nt + .endfn CreateToolhelp32Snapshot,globl + .previous diff --git a/libc/nt/kernel32/Process32FirstW.s b/libc/nt/kernel32/Process32FirstW.s index b8408bce5..d65af4e3f 100644 --- a/libc/nt/kernel32/Process32FirstW.s +++ b/libc/nt/kernel32/Process32FirstW.s @@ -1,2 +1,12 @@ .include "o/libc/nt/codegen.inc" -.imp kernel32,__imp_Process32FirstW,Process32FirstW,1065 +.imp kernel32,__imp_Process32FirstW,Process32FirstW,0 + + .text.windows +Process32First: + push %rbp + mov %rsp,%rbp + .profilable + mov __imp_Process32FirstW(%rip),%rax + jmp __sysv2nt + .endfn Process32First,globl + .previous diff --git a/libc/nt/kernel32/Process32NextW.s b/libc/nt/kernel32/Process32NextW.s index aa49537b1..9ae46191c 100644 --- a/libc/nt/kernel32/Process32NextW.s +++ b/libc/nt/kernel32/Process32NextW.s @@ -1,2 +1,12 @@ .include "o/libc/nt/codegen.inc" -.imp kernel32,__imp_Process32NextW,Process32NextW,1067 +.imp kernel32,__imp_Process32NextW,Process32NextW,0 + + .text.windows +Process32Next: + push %rbp + mov %rsp,%rbp + .profilable + mov __imp_Process32NextW(%rip),%rax + jmp __sysv2nt + .endfn Process32Next,globl + .previous diff --git a/libc/nt/master.sh b/libc/nt/master.sh index 091076566..a46644acf 100755 --- a/libc/nt/master.sh +++ b/libc/nt/master.sh @@ -208,7 +208,7 @@ imp 'CreateThreadpoolWait' CreateThreadpoolWait kernel32 0 imp 'CreateThreadpoolWork' CreateThreadpoolWork kernel32 0 imp 'CreateTimerQueue' CreateTimerQueue kernel32 0 imp 'CreateTimerQueueTimer' CreateTimerQueueTimer kernel32 0 -imp 'CreateToolhelp32Snapshot' CreateToolhelp32Snapshot kernel32 250 +imp 'CreateToolhelp32Snapshot' CreateToolhelp32Snapshot kernel32 0 2 imp 'CreateUmsCompletionList' CreateUmsCompletionList kernel32 251 imp 'CreateUmsThreadContext' CreateUmsThreadContext kernel32 252 imp 'CreateWaitableTimer' CreateWaitableTimerW kernel32 0 3 @@ -934,8 +934,8 @@ imp 'PowerSetRequest' PowerSetRequest kernel32 1059 imp 'PrefetchVirtualMemory' PrefetchVirtualMemory kernel32 0 4 imp 'PrepareTape' PrepareTape kernel32 1061 imp 'PrivMoveFileIdentity' PrivMoveFileIdentityW kernel32 1063 -imp 'Process32First' Process32FirstW kernel32 1065 -imp 'Process32Next' Process32NextW kernel32 1067 +imp 'Process32First' Process32FirstW kernel32 0 2 +imp 'Process32Next' Process32NextW kernel32 0 2 imp 'ProcessIdToSessionId' ProcessIdToSessionId kernel32 0 imp 'PssCaptureSnapshot' PssCaptureSnapshot kernel32 0 imp 'PssDuplicateSnapshot' PssDuplicateSnapshot kernel32 0 diff --git a/libc/nt/process.h b/libc/nt/process.h index 480e1b851..38745121e 100644 --- a/libc/nt/process.h +++ b/libc/nt/process.h @@ -1,6 +1,7 @@ #ifndef COSMOPOLITAN_LIBC_NT_PROCESS_H_ #define COSMOPOLITAN_LIBC_NT_PROCESS_H_ #include "libc/nt/startupinfo.h" +#include "libc/nt/struct/processentry32.h" #include "libc/nt/struct/processinformation.h" #include "libc/nt/struct/processmemorycounters.h" #include "libc/nt/struct/securityattributes.h" @@ -73,6 +74,10 @@ bool32 GetProcessMemoryInfo( int64_t hProcess, struct NtProcessMemoryCountersEx *out_ppsmemCounters, uint32_t cb); +int64_t CreateToolhelp32Snapshot(uint32_t dwFlags, uint32_t th32ProcessID); +bool32 Process32First(int64_t hSnapshot, struct NtProcessEntry32 *in_out_lppe); +bool32 Process32Next(int64_t hSnapshot, struct NtProcessEntry32 *out_lppe); + #if ShouldUseMsabiAttribute() #include "libc/nt/thunk/process.inc" #endif /* ShouldUseMsabiAttribute() */ diff --git a/libc/nt/struct/processentry32.h b/libc/nt/struct/processentry32.h new file mode 100644 index 000000000..e6b481573 --- /dev/null +++ b/libc/nt/struct/processentry32.h @@ -0,0 +1,19 @@ +#ifndef COSMOPOLITAN_LIBC_NT_STRUCT_PROCESSENTRY32_H_ +#define COSMOPOLITAN_LIBC_NT_STRUCT_PROCESSENTRY32_H_ +#if !(__ASSEMBLER__ + __LINKER__ + 0) + +struct NtProcessEntry32 { + uint32_t dwSize; + uint32_t cntUsage; /* unused */ + uint32_t th32ProcessID; + uint64_t th32DefaultHeapID; /* unused */ + uint32_t th32ModuleID; /* unused */ + uint32_t cntThreads; + uint32_t th32ParentProcessID; + int32_t cPriClassBase; + uint32_t dwFlags; /* unused */ + char16_t szExeFile[260]; +}; + +#endif /* !(__ASSEMBLER__ + __LINKER__ + 0) */ +#endif /* COSMOPOLITAN_LIBC_NT_STRUCT_PROCESSENTRY32_H_ */ diff --git a/libc/runtime/clone.greg.c b/libc/runtime/clone.c similarity index 68% rename from libc/runtime/clone.greg.c rename to libc/runtime/clone.c index 02af5b940..5a7b87b37 100644 --- a/libc/runtime/clone.greg.c +++ b/libc/runtime/clone.c @@ -24,7 +24,6 @@ #include "libc/dce.h" #include "libc/errno.h" #include "libc/intrin/asan.internal.h" -#include "libc/intrin/kprintf.h" #include "libc/intrin/spinlock.h" #include "libc/nexgen32e/threaded.h" #include "libc/nt/runtime.h" @@ -50,6 +49,9 @@ STATIC_YOINK("gettid"); // for kprintf() #define LWP_DETACHED 0x00000040 #define LWP_SUSPENDED 0x00000080 +__msabi extern typeof(TlsSetValue) *const __imp_TlsSetValue; +__msabi extern typeof(ExitThread) *const __imp_ExitThread; + struct CloneArgs { union { int tid; @@ -57,12 +59,11 @@ struct CloneArgs { int64_t tid64; }; int lock; - int flags; int *ctid; + int *ztid; char *tls; int (*func)(void *); void *arg; - void *pad; // TODO: Why does FreeBSD clobber this? }; struct __tfork { @@ -76,31 +77,33 @@ static char tibdefault[64]; //////////////////////////////////////////////////////////////////////////////// // THE NEW TECHNOLOGY -uint32_t WinThreadThunk(void *warg); -asm(".section\t.text.windows,\"ax\",@progbits\n\t" - ".local\tWinThreadThunk\n" - "WinThreadThunk:\n\t" - "xor\t%ebp,%ebp\n\t" - "mov\t%rcx,%rdi\n\t" - "mov\t%rcx,%rsp\n\t" - "and\t$-16,%rsp\n\t" - "push\t%rax\n\t" - "jmp\tWinThreadMain\n\t" - ".size\tWinThreadThunk,.-WinThreadThunk\n\t" - ".previous"); -__attribute__((__used__, __no_reorder__)) +int WinThreadLaunch(void *arg, int (*func)(void *), intptr_t rsp); -static textwindows wontreturn void -WinThreadMain(struct CloneArgs *wt) { +// we can't log this function because: +// 1. windows owns the backtrace pointer right now +// 2. ftrace unwinds rbp to determine depth +// we can't use address sanitizer because: +// 1. __asan_handle_no_return wipes stack +// 2. windows owns the stack memory right now +// we need win32 raw imports because: +// 1. generated thunks are function logged +noasan noinstrument static textwindows wontreturn void WinThreadEntry( + int rdi, int rsi, int rdx, struct CloneArgs *wt) { int rc; - if (wt->flags & CLONE_SETTLS) { - TlsSetValue(__tls_index, wt->tls); + if (wt->tls) { + asm("mov\t%1,%%gs:%0" + : "=m"(*((long *)0x1480 + __tls_index)) + : "r"(wt->tls)); } - if (wt->flags & CLONE_CHILD_SETTID) { - *wt->ctid = wt->tid; - } - rc = wt->func(wt->arg); - _Exit1(rc); + *wt->ctid = wt->tid; + rc = WinThreadLaunch(wt->arg, wt->func, (intptr_t)wt & -16); + // we can now clear ctid directly since we're no longer using our own + // stack memory, which can now be safely free'd by the parent thread. + *wt->ztid = 0; + // since we didn't indirect this function through NT2SYSV() it's not + // safe to simply return, and as such, we just call ExitThread(). + __imp_ExitThread(rc); + unreachable; } static textwindows int CloneWindows(int (*func)(void *), char *stk, @@ -111,12 +114,12 @@ static textwindows int CloneWindows(int (*func)(void *), char *stk, wt = (struct CloneArgs *)(((intptr_t)(stk + stksz) - sizeof(struct CloneArgs)) & -alignof(struct CloneArgs)); - wt->flags = flags; - wt->ctid = ctid; + wt->ctid = flags & CLONE_CHILD_SETTID ? ctid : &wt->tid; + wt->ztid = flags & CLONE_CHILD_CLEARTID ? ctid : &wt->tid; wt->func = func; wt->arg = arg; - wt->tls = tls; - if ((h = CreateThread(0, 0, WinThreadThunk, wt, 0, &wt->utid))) { + wt->tls = flags & CLONE_SETTLS ? tls : 0; + if ((h = CreateThread(0, 0, (void *)WinThreadEntry, wt, 0, &wt->utid))) { CloseHandle(h); return wt->tid; } else { @@ -128,7 +131,7 @@ static textwindows int CloneWindows(int (*func)(void *), char *stk, // XNU'S NOT UNIX void XnuThreadThunk(void *pthread, int machport, void *(*func)(void *), - void *arg, intptr_t *stack, unsigned flags); + void *arg, intptr_t *stack, unsigned xnuflags); asm(".local\tXnuThreadThunk\n" "XnuThreadThunk:\n\t" "xor\t%ebp,%ebp\n\t" @@ -141,11 +144,11 @@ __attribute__((__used__, __no_reorder__)) static wontreturn void XnuThreadMain(void *pthread, int tid, int (*func)(void *arg), void *arg, - struct CloneArgs *wt, unsigned flags) { + struct CloneArgs *wt, unsigned xnuflags) { int ax; wt->tid = tid; _spunlock(&wt->lock); - if (wt->flags & CLONE_SETTLS) { + if (wt->tls) { // XNU uses the same 0x30 offset as the WIN32 TIB x64. They told the // Go team at Google that they Apply stands by our ability to use it // https://github.com/golang/go/issues/23617#issuecomment-376662373 @@ -154,10 +157,21 @@ XnuThreadMain(void *pthread, int tid, int (*func)(void *arg), void *arg, : "0"(__NR_thread_fast_set_cthread_self), "D"(wt->tls - 0x30) : "rcx", "r11", "memory", "cc"); } - if (wt->flags & CLONE_CHILD_SETTID) { - *wt->ctid = tid; - } - _Exit1(func(arg)); + *wt->ctid = tid; + func(arg); + // we no longer use the stack after this point + // %rax = int bsdthread_terminate(%rdi = void *stackaddr, + // %rsi = size_t freesize, + // %rdx = uint32_t port, + // %r10 = uint32_t sem); + asm volatile("movl\t$0,%0\n\t" // *wt->ztid = 0 + "xor\t%%r10d,%%r10d\n\t" // sem = 0 + "syscall\n\t" // _Exit1() + "ud2" + : "=m"(*wt->ztid) + : "a"(0x2000000 | 361), "D"(0), "S"(0), "d"(0) + : "rcx", "r10", "r11", "memory"); + unreachable; } static int CloneXnu(int (*fn)(void *), char *stk, size_t stksz, int flags, @@ -180,9 +194,9 @@ static int CloneXnu(int (*fn)(void *), char *stk, size_t stksz, int flags, wt = (struct CloneArgs *)(((intptr_t)(stk + stksz) - sizeof(struct CloneArgs)) & -alignof(struct CloneArgs)); - wt->flags = flags; - wt->ctid = ctid; - wt->tls = tls; + wt->ctid = flags & CLONE_CHILD_SETTID ? ctid : &wt->tid; + wt->ztid = flags & CLONE_CHILD_CLEARTID ? ctid : &wt->tid; + wt->tls = flags & CLONE_SETTLS ? tls : 0; _seizelock(&wt->lock); // TODO: How can we get the tid without locking? if ((rc = bsdthread_create(fn, arg, wt, 0, PTHREAD_START_CUSTOM_XNU)) != -1) { _spinlock(&wt->lock); @@ -194,23 +208,18 @@ static int CloneXnu(int (*fn)(void *), char *stk, size_t stksz, int flags, //////////////////////////////////////////////////////////////////////////////// // FREE BESIYATA DISHMAYA -void FreebsdThreadThunk(void *) wontreturn; -asm(".local\tFreebsdThreadThunk\n" - "FreebsdThreadThunk:\n\t" - "xor\t%ebp,%ebp\n\t" - "mov\t%rdi,%rsp\n\t" - "and\t$-16,%rsp\n\t" - "push\t%rax\n\t" - "jmp\tFreebsdThreadMain\n\t" - ".size\tFreebsdThreadThunk,.-FreebsdThreadThunk"); -__attribute__((__used__, __no_reorder__)) - -static wontreturn void -FreebsdThreadMain(struct CloneArgs *wt) { - if (wt->flags & CLONE_CHILD_SETTID) { - *wt->ctid = wt->tid; - } - _Exit1(wt->func(wt->arg)); +static wontreturn void FreebsdThreadMain(void *p) { + struct CloneArgs *wt = p; + *wt->ctid = wt->tid; + wt->func(wt->arg); + // we no longer use the stack after this point + // void thr_exit(%rdi = long *state); + asm volatile("movl\t$0,%0\n\t" // *wt->ztid = 0 + "syscall" // _Exit1() + : "=m"(*wt->ztid) + : "a"(431), "D"(0) + : "rcx", "r11", "memory"); + unreachable; } static int CloneFreebsd(int (*func)(void *), char *stk, size_t stksz, int flags, @@ -222,16 +231,16 @@ static int CloneFreebsd(int (*func)(void *), char *stk, size_t stksz, int flags, wt = (struct CloneArgs *)(((intptr_t)(stk + stksz) - sizeof(struct CloneArgs)) & -alignof(struct CloneArgs)); - wt->flags = flags; - wt->ctid = ctid; + wt->ctid = flags & CLONE_CHILD_SETTID ? ctid : &wt->tid; + wt->ztid = flags & CLONE_CHILD_CLEARTID ? ctid : &wt->tid; wt->tls = tls; wt->func = func; wt->arg = arg; struct thr_param params = { - .start_func = FreebsdThreadThunk, + .start_func = FreebsdThreadMain, .arg = wt, .stack_base = stk, - .stack_size = stksz, + .stack_size = (((intptr_t)wt - (intptr_t)stk) & -16) - 8, .tls_base = flags & CLONE_SETTLS ? tls : 0, .tls_size = flags & CLONE_SETTLS ? tlssz : 0, .child_tid = &wt->tid64, @@ -277,7 +286,15 @@ __attribute__((__used__, __no_reorder__)) static privileged wontreturn void OpenbsdThreadMain(struct CloneArgs *wt) { - _Exit1(wt->func(wt->arg)); + wt->func(wt->arg); + // we no longer use the stack after this point + // void __threxit(%rdi = int32_t *notdead); + asm volatile("movl\t$0,%0\n\t" // *wt->ztid = 0 + "syscall" // _Exit1() + : "=m"(*wt->ztid) + : "a"(302), "D"(0) + : "rcx", "r11", "memory"); + unreachable; } static int CloneOpenbsd(int (*func)(void *), char *stk, size_t stksz, int flags, @@ -288,8 +305,8 @@ static int CloneOpenbsd(int (*func)(void *), char *stk, size_t stksz, int flags, wt = (struct CloneArgs *)(((intptr_t)(stk + stksz) - sizeof(struct CloneArgs)) & -alignof(struct CloneArgs)); - wt->flags = flags; - wt->ctid = ctid; + wt->ctid = flags & CLONE_CHILD_SETTID ? ctid : &wt->tid; + wt->ztid = flags & CLONE_CHILD_CLEARTID ? ctid : &wt->tid; wt->func = func; wt->arg = arg; params.tf_stack = wt; @@ -306,12 +323,19 @@ static int CloneOpenbsd(int (*func)(void *), char *stk, size_t stksz, int flags, // NET BESIYATA DISHMAYA static wontreturn void NetbsdThreadMain(void *arg, int (*func)(void *arg), - int *tid, int *ctid, int flags) { - int rc; - if (flags & CLONE_CHILD_SETTID) { - *ctid = *tid; - } - _Exit1(func(arg)); + int *tid, int *ctid, int *ztid) { + int ax, dx; + *ctid = *tid; + func(arg); + // we no longer use the stack after this point + // %eax = int __lwp_exit(void); + asm volatile("movl\t$0,%2\n\t" // *wt->ztid = 0 + "syscall\n\t" // _Exit1() + "ud2" + : "=a"(ax), "=d"(dx), "=m"(*ztid) + : "0"(310) + : "rcx", "r11", "memory"); + unreachable; } static int CloneNetbsd(int (*func)(void *), char *stk, size_t stksz, int flags, @@ -325,8 +349,10 @@ static int CloneNetbsd(int (*func)(void *), char *stk, size_t stksz, int flags, intptr_t dx, sp; static bool once; static int broken; - struct ucontext_netbsd *ctx; + struct ucontext_netbsd ctx; static struct ucontext_netbsd netbsd_clone_template; + + // memoize arbitrary valid processor state structure if (!once) { asm volatile(CFLAG_ASM("syscall") : CFLAG_CONSTRAINT(failed), "=a"(ax) @@ -343,31 +369,34 @@ static int CloneNetbsd(int (*func)(void *), char *stk, size_t stksz, int flags, } sp = (intptr_t)(stk + stksz); sp -= sizeof(int); + sp = sp & -alignof(int); tid = (int *)sp; - sp -= sizeof(*ctx); - sp = sp & -alignof(*ctx); - ctx = (struct ucontext_netbsd *)sp; - memcpy(ctx, &netbsd_clone_template, sizeof(*ctx)); - ctx->uc_link = 0; - ctx->uc_mcontext.rbp = 0; - ctx->uc_mcontext.rsp = sp; - ctx->uc_mcontext.rip = (intptr_t)NetbsdThreadMain; - ctx->uc_mcontext.rdi = (intptr_t)arg; - ctx->uc_mcontext.rsi = (intptr_t)func; - ctx->uc_mcontext.rdx = (intptr_t)tid; - ctx->uc_mcontext.rcx = (intptr_t)ctid; - ctx->uc_mcontext.r8 = flags; - ctx->uc_flags |= _UC_STACK; - ctx->uc_stack.ss_sp = stk; - ctx->uc_stack.ss_size = stksz; - ctx->uc_stack.ss_flags = 0; + sp = sp & -16; + sp -= 8; + // pass parameters in process state + memcpy(&ctx, &netbsd_clone_template, sizeof(ctx)); + ctx.uc_link = 0; + ctx.uc_mcontext.rbp = 0; + ctx.uc_mcontext.rsp = sp; + ctx.uc_mcontext.rip = (intptr_t)NetbsdThreadMain; + ctx.uc_mcontext.rdi = (intptr_t)arg; + ctx.uc_mcontext.rsi = (intptr_t)func; + ctx.uc_mcontext.rdx = (intptr_t)tid; + ctx.uc_mcontext.rcx = (intptr_t)(flags & CLONE_CHILD_SETTID ? ctid : tid); + ctx.uc_mcontext.r8 = (intptr_t)(flags & CLONE_CHILD_CLEARTID ? ctid : tid); + ctx.uc_flags |= _UC_STACK; + ctx.uc_stack.ss_sp = stk; + ctx.uc_stack.ss_size = stksz; + ctx.uc_stack.ss_flags = 0; if (flags & CLONE_SETTLS) { - ctx->uc_flags |= _UC_TLSBASE; - ctx->uc_mcontext._mc_tlsbase = (intptr_t)tls; + ctx.uc_flags |= _UC_TLSBASE; + ctx.uc_mcontext._mc_tlsbase = (intptr_t)tls; } + + // perform the system call asm volatile(CFLAG_ASM("syscall") : CFLAG_CONSTRAINT(failed), "=a"(ax), "=d"(dx) - : "1"(__NR__lwp_create), "D"(ctx), "S"(LWP_DETACHED), "2"(tid) + : "1"(__NR__lwp_create), "D"(&ctx), "S"(LWP_DETACHED), "2"(tid) : "rcx", "r11", "memory"); if (!failed) { return *tid; @@ -388,6 +417,12 @@ int CloneLinux(int (*func)(void *), char *stk, size_t stksz, int flags, int ax; intptr_t *stack = (intptr_t *)(stk + stksz); *--stack = (intptr_t)arg; + // %rax = syscall(%rax = __NR_clone, + // %rdi = flags, + // %rsi = child_stack, + // %rdx = parent_tidptr, + // %r10 = child_tidptr, + // %r8 = new_tls); asm volatile("mov\t%4,%%r10\n\t" // ctid "mov\t%5,%%r8\n\t" // tls "mov\t%6,%%r9\n\t" // func @@ -398,10 +433,11 @@ int CloneLinux(int (*func)(void *), char *stk, size_t stksz, int flags, "pop\t%%rdi\n\t" // arg "call\t*%%r9\n\t" // func "xchg\t%%eax,%%edi\n\t" - "jmp\t_Exit1\n1:" + "mov\t$0x3c,%%eax\n\t" + "syscall\n1:" : "=a"(ax) : "0"(__NR_clone_linux), "D"(flags), "S"(stack), "g"(ctid), - "g"(tls), "g"(func) + "g"(tls), "g"(func), "d"(ptid) : "rcx", "r8", "r9", "r10", "r11", "memory"); if (ax > -4096u) errno = -ax, ax = -1; return ax; @@ -419,8 +455,7 @@ int CloneLinux(int (*func)(void *), char *stk, size_t stksz, int flags, * function should be synchronized using shared memory operations. * * Any memory that's required by this system call wrapper is allocated - * to the top of your stack. This is normally about 64 bytes, although - * on NetBSD it's currently 800. + * to the top of your stack. This shouldn't be more than 128 bytes. * * Your function is called from within the stack you specify. A return * address is pushed onto your stack, that causes returning to jump to @@ -464,9 +499,25 @@ int CloneLinux(int (*func)(void *), char *stk, size_t stksz, int flags, * and it's advised to have the bottom-most page, be a guard page * @param flags should have: * - `CLONE_THREAD|CLONE_VM|CLONE_FS|CLONE_FILES|CLONE_SIGHAND` - * and may optionally bitwise any of the following: - * - `CLONE_CHILD_SETTID` is needed too if you use `ctid` - * - `CLONE_SETTLS` is needed too if you set `tls` + * and you may optionally bitwise or any of the following: + * - `CLONE_CHILD_SETTID` is needed too if you use `ctid` which + * is part of the memory the child owns and it'll be set right + * before the callback function is invoked + * - `CLONE_CHILD_CLEARTID` causes `*ctid = 0` upon termination + * which can be used to implement join so that the parent may + * safely free the stack memory that the child is using + * - `CLONE_PARENT_SETTID` is needed too if you use `ptid` and this + * is guaranteed to happen before clone() returns + * - `CLONE_SETTLS` is needed too if you set `tls`. You may get this + * value from the thread by calling __get_tls(). There are a few + * layout expectations imposed by your C library. Those are all + * documented by __initialize_tls() which initializes the parts of + * the first 64 bytes of tls memory that libc cares about. Also + * note that if you decide to use tls once then you must use it + * for everything, since this flag also flips a runtime state that + * enables it for the main thread and functions such as + * __errno_location() will begin assuming they can safely access + * the tls segment register. * @param arg will be passed to your callback * @param tls may be used to set the thread local storage segment; * this parameter is ignored if `CLONE_SETTLS` is not set @@ -506,10 +557,12 @@ int clone(int (*func)(void *), void *stk, size_t stksz, int flags, void *arg, rc = einval(); } else if (IsLinux()) { rc = CloneLinux(func, stk, stksz, flags, arg, ptid, tls, tlssz, ctid); - } else if (!IsTiny() && (flags & ~(CLONE_SETTLS | CLONE_PARENT_SETTID | - CLONE_CHILD_SETTID)) != - (CLONE_THREAD | CLONE_VM | CLONE_FS | - CLONE_FILES | CLONE_SIGHAND)) { + } else if (!IsTiny() && + (flags & ~(CLONE_SETTLS | CLONE_PARENT_SETTID | + CLONE_CHILD_SETTID | CLONE_CHILD_CLEARTID)) != + (CLONE_THREAD | CLONE_VM | CLONE_FS | CLONE_FILES | + CLONE_SIGHAND)) { + STRACE("clone flag unsupported on this platform"); rc = einval(); } else if (IsXnu()) { rc = CloneXnu(func, stk, stksz, flags, arg, tls, tlssz, ctid); @@ -525,7 +578,12 @@ int clone(int (*func)(void *), void *stk, size_t stksz, int flags, void *arg, rc = enosys(); } - STRACE("clone(%p, %p, %'zu, %#x, %p, %p, %p, %'zu, %p) → %d", func, stk, + if (rc != -1 && (flags & CLONE_PARENT_SETTID)) { + *ptid = rc; + } + + STRACE("clone(%p, %p, %'zu, %#x, %p, %p, %p, %'zu, %p) → %d% m", func, stk, stksz, flags, arg, ptid, tls, tlssz, ctid, rc); + return rc; } diff --git a/libc/runtime/ftracer.c b/libc/runtime/ftracer.c index c3494741a..feb47416f 100644 --- a/libc/runtime/ftracer.c +++ b/libc/runtime/ftracer.c @@ -17,9 +17,11 @@ │ PERFORMANCE OF THIS SOFTWARE. │ ╚─────────────────────────────────────────────────────────────────────────────*/ #include "libc/bits/safemacros.internal.h" +#include "libc/calls/calls.h" #include "libc/fmt/itoa.h" #include "libc/intrin/cmpxchg.h" #include "libc/intrin/kprintf.h" +#include "libc/intrin/lockcmpxchgp.h" #include "libc/intrin/spinlock.h" #include "libc/log/libfatal.internal.h" #include "libc/macros.internal.h" @@ -49,7 +51,7 @@ void ftrace_hook(void); -_Alignas(64) char ftrace_lock; +_Alignas(64) int ftrace_lock; static struct Ftrace { int skew; @@ -75,6 +77,32 @@ static privileged int GetNestingLevel(struct StackFrame *frame) { return MIN(MAX_NESTING, nesting); } +static privileged inline void ReleaseFtraceLock(void) { + int zero = 0; + __atomic_store(&ftrace_lock, &zero, __ATOMIC_RELAXED); +} + +static privileged inline bool AcquireFtraceLock(void) { + int me, owner, tries; + for (tries = 0, me = gettid();;) { + owner = 0; + if (_lockcmpxchgp(&ftrace_lock, &owner, me)) { + return true; + } + if (owner == me) { + // we ignore re-entry into ftrace. while the code and build config + // is written to make re-entry highly unlikely, it's impossible to + // guarantee. there's also the possibility of asynchronous signals + return false; + } + if (++tries & 7) { + __builtin_ia32_pause(); + } else { + sched_yield(); + } + } +} + /** * Prints name of function being called. * @@ -83,21 +111,22 @@ static privileged int GetNestingLevel(struct StackFrame *frame) { * according to the System Five NexGen32e ABI. */ privileged void ftracer(void) { + long stackuse; uint64_t stamp; - size_t stackuse; struct StackFrame *frame; - _spinlock_cooperative(&ftrace_lock); - stamp = rdtsc(); - frame = __builtin_frame_address(0); - frame = frame->next; - if (frame->addr != g_ftrace.lastaddr) { - stackuse = ROUNDUP((intptr_t)frame, GetStackSize()) - (intptr_t)frame; - kprintf("%rFUN %5P %'13T %'*lu %*s%t\r\n", g_ftrace.stackdigs, stackuse, - GetNestingLevel(frame) * 2, "", frame->addr); - g_ftrace.laststamp = X86_HAVE(RDTSCP) ? rdtscp(0) : rdtsc(); - g_ftrace.lastaddr = frame->addr; + if (AcquireFtraceLock()) { + stamp = rdtsc(); + frame = __builtin_frame_address(0); + frame = frame->next; + if (frame->addr != g_ftrace.lastaddr) { + stackuse = (intptr_t)GetStackAddr(0) + GetStackSize() - (intptr_t)frame; + kprintf("%rFUN %5P %'13T %'*ld %*s%t\r\n", g_ftrace.stackdigs, stackuse, + GetNestingLevel(frame) * 2, "", frame->addr); + g_ftrace.laststamp = X86_HAVE(RDTSCP) ? rdtscp(0) : rdtsc(); + g_ftrace.lastaddr = frame->addr; + } + ReleaseFtraceLock(); } - _spunlock(&ftrace_lock); } textstartup int ftrace_install(void) { diff --git a/libc/runtime/getsymboltable.greg.c b/libc/runtime/getsymboltable.c similarity index 88% rename from libc/runtime/getsymboltable.greg.c rename to libc/runtime/getsymboltable.c index 38e54f0d4..fa0358e36 100644 --- a/libc/runtime/getsymboltable.greg.c +++ b/libc/runtime/getsymboltable.c @@ -31,7 +31,7 @@ #include "libc/zipos/zipos.internal.h" static char g_lock; -static struct SymbolTable *g_symtab; +hidden struct SymbolTable *__symtab; // for kprintf /** * Looks for `.symtab` in zip central directory. @@ -70,6 +70,7 @@ static struct SymbolTable *GetSymbolTableFromZip(struct Zipos *zipos) { memcpy(res, (void *)ZIP_LFILE_CONTENT(zipos->map + lf), size); break; #if 0 + // TODO(jart): fix me case kZipCompressionDeflate: rc = undeflate(res, size, (void *)ZIP_LFILE_CONTENT(zipos->map + lf), GetZipLfileCompressedSize(zipos->map + lf), &ds); @@ -121,21 +122,21 @@ static struct SymbolTable *GetSymbolTableFromElf(void) { struct SymbolTable *GetSymbolTable(void) { struct Zipos *z; if (_trylock(&g_lock)) return 0; - if (!g_symtab && !__isworker) { + if (!__symtab && !__isworker) { if (weaken(__zipos_get) && (z = weaken(__zipos_get)())) { - if ((g_symtab = GetSymbolTableFromZip(z))) { - g_symtab->names = - (uint32_t *)((char *)g_symtab + g_symtab->names_offset); - g_symtab->name_base = - (char *)((char *)g_symtab + g_symtab->name_base_offset); + if ((__symtab = GetSymbolTableFromZip(z))) { + __symtab->names = + (uint32_t *)((char *)__symtab + __symtab->names_offset); + __symtab->name_base = + (char *)((char *)__symtab + __symtab->name_base_offset); } } - if (!g_symtab) { - g_symtab = GetSymbolTableFromElf(); + if (!__symtab) { + __symtab = GetSymbolTableFromElf(); } } _spunlock(&g_lock); - return g_symtab; + return __symtab; } /** @@ -144,11 +145,14 @@ struct SymbolTable *GetSymbolTable(void) { * @param t if null will be auto-populated only if already open * @return index or -1 if nothing found */ -privileged int __get_symbol(struct SymbolTable *t, intptr_t a) { - /* asan runtime depends on this function */ +noinstrument privileged int __get_symbol(struct SymbolTable *t, intptr_t a) { + // we need privileged because: + // kprintf is privileged and it depends on this + // we don't want function tracing because: + // function tracing depends on this function via kprintf unsigned l, m, r, n, k; - if (!t && g_symtab) { - t = g_symtab; + if (!t && __symtab) { + t = __symtab; } if (t) { l = 0; diff --git a/libc/runtime/memtrack.greg.c b/libc/runtime/memtrack.greg.c index e4f58c759..9ea60d1e1 100644 --- a/libc/runtime/memtrack.greg.c +++ b/libc/runtime/memtrack.greg.c @@ -39,7 +39,7 @@ static void *MoveMemoryIntervals(struct MemoryInterval *d, const struct MemoryInterval *s, int n) { - /* asan runtime depends on this function */ + // asan runtime depends on this function int i; assert(n >= 0); if (d > s) { @@ -55,7 +55,7 @@ static void *MoveMemoryIntervals(struct MemoryInterval *d, } static void RemoveMemoryIntervals(struct MemoryIntervals *mm, int i, int n) { - /* asan runtime depends on this function */ + // asan runtime depends on this function assert(i >= 0); assert(i + n <= mm->i); MoveMemoryIntervals(mm->p + i, mm->p + i + n, mm->i - (i + n)); @@ -71,7 +71,7 @@ static bool ExtendMemoryIntervals(struct MemoryIntervals *mm) { base = (char *)kMemtrackStart; prot = PROT_READ | PROT_WRITE; flags = MAP_ANONYMOUS | MAP_PRIVATE | MAP_FIXED; - /* TODO(jart): These map handles should not leak across NT fork() */ + // TODO(jart): These map handles should not leak across NT fork() if (mm->p == mm->s) { if (IsAsan()) { shad = (char *)(((intptr_t)base >> 3) + 0x7fff8000); @@ -100,7 +100,7 @@ static bool ExtendMemoryIntervals(struct MemoryIntervals *mm) { } int CreateMemoryInterval(struct MemoryIntervals *mm, int i) { - /* asan runtime depends on this function */ + // asan runtime depends on this function int rc; rc = 0; assert(i >= 0); @@ -192,7 +192,7 @@ int ReleaseMemoryIntervals(struct MemoryIntervals *mm, int x, int y, int TrackMemoryInterval(struct MemoryIntervals *mm, int x, int y, long h, int prot, int flags, bool readonlyfile, bool iscow, long offset, long size) { - /* asan runtime depends on this function */ + // asan runtime depends on this function unsigned i; assert(y >= x); assert(AreMemoryIntervalsOk(mm)); diff --git a/libc/runtime/memtrack.internal.h b/libc/runtime/memtrack.internal.h index 72c3fb3e3..43e9efacd 100644 --- a/libc/runtime/memtrack.internal.h +++ b/libc/runtime/memtrack.internal.h @@ -1,6 +1,7 @@ #ifndef COSMOPOLITAN_LIBC_RUNTIME_MEMTRACK_H_ #define COSMOPOLITAN_LIBC_RUNTIME_MEMTRACK_H_ #include "libc/assert.h" +#include "libc/bits/midpoint.h" #include "libc/dce.h" #include "libc/macros.internal.h" #include "libc/nt/enum/version.h" @@ -168,7 +169,7 @@ forceinline unsigned FindMemoryInterval(const struct MemoryIntervals *mm, l = 0; r = mm->i; while (l < r) { - m = (l + r) >> 1; + m = _midpoint(l, r); if (mm->p[m].y < x) { l = m + 1; } else { diff --git a/libc/runtime/mmap.c b/libc/runtime/mmap.c index d9e0337ab..cb58ffb8c 100644 --- a/libc/runtime/mmap.c +++ b/libc/runtime/mmap.c @@ -68,57 +68,83 @@ static wontreturn void OnUnrecoverableMmapError(const char *s) { _Exit(199); } -noasan static bool IsMapped(char *p, size_t n) { - return OverlapsImageSpace(p, n) || IsMemtracked(FRAME(p), FRAME(p + (n - 1))); +static noasan inline bool OverlapsExistingMapping(char *p, size_t n) { + int a, b, i; + assert(n > 0); + a = FRAME(p); + b = FRAME(p + (n - 1)); + i = FindMemoryInterval(&_mmi, a); + if (i < _mmi.i) { + if (a <= _mmi.p[i].x && _mmi.p[i].x <= b) return true; + if (a <= _mmi.p[i].y && _mmi.p[i].y <= b) return true; + if (_mmi.p[i].x <= a && b <= _mmi.p[i].y) return true; + } + return false; } -noasan static bool NeedAutomap(char *p, size_t n) { - return !p || OverlapsArenaSpace(p, n) || OverlapsShadowSpace(p, n) || - IsMapped(p, n); -} - -noasan static bool ChooseMemoryInterval(int x, int n, int *res) { - int i; +static noasan bool ChooseMemoryInterval(int x, int n, int align, int *res) { + int i, start, end; + assert(align > 0); if (_mmi.i) { + + // find the start of the automap memory region i = FindMemoryInterval(&_mmi, x); if (i < _mmi.i) { - if (x + n < _mmi.p[i].x) { - *res = x; - return true; + + // check to see if there's space available before the first entry + if (!__builtin_add_overflow(x, align - 1, &start)) { + start &= -align; + if (!__builtin_add_overflow(start, n - 1, &end)) { + if (end < _mmi.p[i].x) { + *res = start; + return true; + } + } } + + // check to see if there's space available between two entries while (++i < _mmi.i) { - if (_mmi.p[i].x - _mmi.p[i - 1].y > n) { - *res = _mmi.p[i - 1].y + 1; - return true; + if (!__builtin_add_overflow(_mmi.p[i - 1].y, 1, &start) && + !__builtin_add_overflow(start, align - 1, &start)) { + start &= -align; + if (!__builtin_add_overflow(start, n - 1, &end)) { + if (end < _mmi.p[i].x) { + *res = start; + return true; + } + } } } } - if (INT_MAX - _mmi.p[i - 1].y >= n) { - *res = _mmi.p[i - 1].y + 1; - return true; + + // otherwise append after the last entry if space is available + if (!__builtin_add_overflow(_mmi.p[i - 1].y, 1, &start) && + !__builtin_add_overflow(start, align - 1, &start)) { + start &= -align; + if (!__builtin_add_overflow(start, n - 1, &end)) { + *res = start; + return true; + } } - return false; + } else { - *res = x; - return true; + // if memtrack is empty, then just assign the requested address + // assuming it doesn't overflow + if (!__builtin_add_overflow(x, align - 1, &start)) { + start &= -align; + if (!__builtin_add_overflow(start, n - 1, &end)) { + *res = start; + return true; + } + } } + + return false; } -noasan static bool Automap(int n, int *res) { - *res = -1; - if (ChooseMemoryInterval(FRAME(kAutomapStart), n, res)) { - assert(*res >= FRAME(kAutomapStart)); - if (*res + n <= FRAME(kAutomapStart + (kAutomapStart - 1))) { - return true; - } else { - STRACE("mmap(%.12p, %p) ENOMEM (automap interval exhausted)", ADDR(*res), - ADDR(n + 1)); - return false; - } - } else { - STRACE("mmap(%.12p, %p) ENOMEM (automap failed)", ADDR(*res), ADDR(n + 1)); - return false; - } +noasan static bool Automap(int count, int align, int *res) { + return ChooseMemoryInterval(FRAME(kAutomapStart), count, align, res) && + *res + count <= FRAME(kAutomapStart + (kAutomapSize - 1)); } noasan static size_t GetMemtrackSize(struct MemoryIntervals *mm) { @@ -221,21 +247,16 @@ static noasan inline void *Mmap(void *addr, size_t size, int prot, int flags, } #endif char *p = addr; - bool needguard; struct DirectMap dm; - size_t virtualused, virtualneed; int a, b, i, f, m, n, x; + bool needguard, clashes; + size_t virtualused, virtualneed; if (UNLIKELY(!size)) { STRACE("size=0"); return VIP(einval()); } - if (UNLIKELY(!IsLegalSize(size))) { - STRACE("size isn't 48-bit"); - return VIP(einval()); - } - if (UNLIKELY(!IsLegalPointer(p))) { STRACE("p isn't 48-bit"); return VIP(einval()); @@ -266,29 +287,28 @@ static noasan inline void *Mmap(void *addr, size_t size, int prot, int flags, return VIP(einval()); } - if (UNLIKELY(INT64_MAX - size < off)) { - STRACE("too large"); - return VIP(einval()); - } - if (UNLIKELY(!ALIGNED(off))) { STRACE("p isn't 64kb aligned"); return VIP(einval()); } - if ((flags & MAP_FIXED_NOREPLACE) && IsMapped(p, size)) { -#ifdef SYSDEBUG - if (OverlapsImageSpace(p, size)) { - STRACE("overlaps image"); - } else { - STRACE("overlaps existing"); + if (fd == -1) { + size = ROUNDUP(size, FRAMESIZE); + if (IsWindows()) { + prot |= PROT_WRITE; /* kludge */ } -#endif - return VIP(efault()); + } else if (__isfdkind(fd, kFdZip)) { + STRACE("fd is zipos handle"); + return VIP(einval()); } - if (__isfdkind(fd, kFdZip)) { - STRACE("fd is zipos handle"); + if (UNLIKELY(!IsLegalSize(size))) { + STRACE("size isn't 48-bit"); + return VIP(einval()); + } + + if (UNLIKELY(INT64_MAX - size < off)) { + STRACE("too large"); return VIP(einval()); } @@ -301,15 +321,29 @@ static noasan inline void *Mmap(void *addr, size_t size, int prot, int flags, return VIP(enomem()); } - if (fd == -1) { - size = ROUNDUP(size, FRAMESIZE); - if (IsWindows()) { - prot |= PROT_WRITE; /* kludge */ - } + clashes = OverlapsImageSpace(p, size) || OverlapsExistingMapping(p, size); + + if ((flags & MAP_FIXED_NOREPLACE) && clashes) { + STRACE("noreplace overlaps existing"); + return VIP(eexist()); + } + + if (__builtin_add_overflow((int)(size >> 16), (int)!!(size & (FRAMESIZE - 1)), + &n)) { + STRACE("memory range overflows"); + return VIP(einval()); + } + + // if size is a two power then automap will use it as alignment + if (IS2POW(size)) { + a = size >> 16; + if (!a) { + a = 1; + } + } else { + a = 1; } - n = (int)(size >> 16) + !!(size & (FRAMESIZE - 1)); - assert(n > 0); f = (flags & ~MAP_FIXED_NOREPLACE) | MAP_FIXED; if (flags & MAP_FIXED) { x = FRAME(p); @@ -318,10 +352,11 @@ static noasan inline void *Mmap(void *addr, size_t size, int prot, int flags, OnUnrecoverableMmapError("FIXED UNTRACK FAILED"); } } - } else if (!NeedAutomap(p, size)) { + } else if (p && !clashes && !OverlapsArenaSpace(p, size) && + !OverlapsShadowSpace(p, size)) { x = FRAME(p); - } else if (!Automap(n, &x)) { - STRACE("AUTOMAP OUT OF MEMORY D:"); + } else if (!Automap(n, a, &x)) { + STRACE("automap has no room for %d frames with %d alignment", n, a); return VIP(enomem()); } diff --git a/libc/runtime/printargs.greg.c b/libc/runtime/printargs.greg.c index 168597e99..55e2f98f1 100644 --- a/libc/runtime/printargs.greg.c +++ b/libc/runtime/printargs.greg.c @@ -337,7 +337,7 @@ textstartup void __printargs(const char *prologue) { PRINT(""); PRINT("MEMTRACK"); - PrintMemoryIntervals(2, &_mmi); + __print_maps(); PRINT(""); PRINT("TERMIOS"); diff --git a/libc/runtime/printmaps.c b/libc/runtime/printmaps.c new file mode 100644 index 000000000..adba80522 --- /dev/null +++ b/libc/runtime/printmaps.c @@ -0,0 +1,30 @@ +/*-*- mode:c;indent-tabs-mode:nil;c-basic-offset:2;tab-width:8;coding:utf-8 -*-│ +│vi: set net ft=c ts=2 sts=2 sw=2 fenc=utf-8 :vi│ +╞══════════════════════════════════════════════════════════════════════════════╡ +│ Copyright 2022 Justine Alexandra Roberts Tunney │ +│ │ +│ Permission to use, copy, modify, and/or distribute this software for │ +│ any purpose with or without fee is hereby granted, provided that the │ +│ above copyright notice and this permission notice appear in all copies. │ +│ │ +│ THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL │ +│ WARRANTIES WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED │ +│ WARRANTIES OF MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE │ +│ AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL │ +│ DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR │ +│ PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER │ +│ TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR │ +│ PERFORMANCE OF THIS SOFTWARE. │ +╚─────────────────────────────────────────────────────────────────────────────*/ +#include "libc/intrin/spinlock.h" +#include "libc/runtime/memtrack.internal.h" +#include "libc/runtime/runtime.h" + +/** + * Prints memory mappings to stderr. + */ +void __print_maps(void) { + _spinlock(&_mmi.lock); + PrintMemoryIntervals(2, &_mmi); + _spunlock(&_mmi.lock); +} diff --git a/libc/runtime/runtime.h b/libc/runtime/runtime.h index a5b4d50cc..f486d4909 100644 --- a/libc/runtime/runtime.h +++ b/libc/runtime/runtime.h @@ -112,6 +112,7 @@ void __morph_begin(void); void __morph_end(void); unsigned char *GetFirstInstruction(void); unsigned char *GetInstructionLengths(void); +void __print_maps(void); COSMOPOLITAN_C_END_ #endif /* !(__ASSEMBLER__ + __LINKER__ + 0) */ diff --git a/libc/runtime/runtime.mk b/libc/runtime/runtime.mk index 63fc71036..43501f3a3 100644 --- a/libc/runtime/runtime.mk +++ b/libc/runtime/runtime.mk @@ -64,6 +64,7 @@ $(LIBC_RUNTIME_A).pkg: \ # this is the function tracing runtime o/$(MODE)/libc/runtime/ftracer.o: \ OVERRIDE_CFLAGS += \ + -x-no-pg \ -mno-fentry \ -ffreestanding \ -fno-sanitize=all @@ -86,8 +87,7 @@ o/$(MODE)/libc/runtime/print.greg.o \ o/$(MODE)/libc/runtime/stackchkfail.o \ o/$(MODE)/libc/runtime/stackchkfaillocal.o \ o/$(MODE)/libc/runtime/winmain.greg.o \ -o/$(MODE)/libc/runtime/opensymboltable.o \ -o/$(MODE)/libc/runtime/getsymboltable.greg.o: \ +o/$(MODE)/libc/runtime/opensymboltable.o: \ OVERRIDE_CFLAGS += \ -Os \ -ffreestanding \ diff --git a/libc/runtime/stack.h b/libc/runtime/stack.h index 4e231d54a..88781ced6 100644 --- a/libc/runtime/stack.h +++ b/libc/runtime/stack.h @@ -12,6 +12,7 @@ * This defaults to `STACKSIZE`. The bottom-most page will be protected * to ensure your stack does not magically grow beyond this value. It's * possible to detect stack overflows, by calling `ShowCrashReports()`. + * Your stack size must be a power of two; the linker will check this. * * If you want to know how much stack your programs needs, then * @@ -28,11 +29,17 @@ /** * Tunes APE stack virtual address. * - * This defaults to `0x7e0000000000 - STACKSIZE`. The value defined by - * this macro will be respected, with two exceptions: (1) in MODE=tiny - * the operating system provided stack is used instead and (2) Windows - * Seven doesn't support 64-bit addresses so 0x10000000 - GetStackSize - * is used instead. + * This value must be aligned according to your stack size, and that's + * checked by your linker script. This defaults to `0x700000000000` so + * + * 1. It's easy to see how close you are to the bottom + * 2. The linker script error is unlikely to happen + * + * This macro will be respected, with two exceptions + * + * 1. In MODE=tiny the operating system provided stack is used instead + * 2. Windows 7 doesn't support 64-bit addresses, so we'll instead use + * `0x10000000 - GetStackSize()` as the stack address * * @see libc/sysv/systemfive.S * @see libc/nt/winmain.greg.c @@ -56,10 +63,20 @@ extern char ape_stack_prot[] __attribute__((__weak__)); extern char ape_stack_memsz[] __attribute__((__weak__)); extern char ape_stack_align[] __attribute__((__weak__)); +/** + * Returns size of stack, which is always a two power. + */ #define GetStackSize() ((uintptr_t)ape_stack_memsz) /** * Returns address of bottom of stack. + * + * This takes into consideration threads and sigaltstack. This is + * implemented as a fast pure expression, since we're able to make the + * assumption that stack sizes are two powers and aligned. This is + * thanks to (1) the linker script checks the statically chosen sizes, + * and (2) the mmap() address picker will choose aligned addresses when + * the provided size is a two power. */ #define GetStackAddr(ADDEND) \ ((void *)((((intptr_t)__builtin_frame_address(0) - 1) & -GetStackSize()) + \ @@ -67,6 +84,12 @@ extern char ape_stack_align[] __attribute__((__weak__)); /** * Returns preferred bottom address of stack. + * + * This is the stakc address of the main process. The only time that + * isn't guaranteed to be the case is in MODE=tiny, since it doesn't + * link the code for stack creation at startup. This generally isn't + * problematic, since MODE=tiny doesn't use any of the runtime codes + * which want the stack to be cheaply knowable, e.g. ftrace, kprintf */ #define GetStaticStackAddr(ADDEND) \ ({ \ diff --git a/libc/runtime/winthreadlaunch.S b/libc/runtime/winthreadlaunch.S new file mode 100644 index 000000000..fc86864be --- /dev/null +++ b/libc/runtime/winthreadlaunch.S @@ -0,0 +1,46 @@ +/*-*- mode:unix-assembly; indent-tabs-mode:t; tab-width:8; coding:utf-8 -*-│ +│vi: set et ft=asm ts=8 tw=8 fenc=utf-8 :vi│ +╞══════════════════════════════════════════════════════════════════════════════╡ +│ Copyright 2022 Justine Alexandra Roberts Tunney │ +│ │ +│ Permission to use, copy, modify, and/or distribute this software for │ +│ any purpose with or without fee is hereby granted, provided that the │ +│ above copyright notice and this permission notice appear in all copies. │ +│ │ +│ THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL │ +│ WARRANTIES WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED │ +│ WARRANTIES OF MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE │ +│ AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL │ +│ DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR │ +│ PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER │ +│ TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR │ +│ PERFORMANCE OF THIS SOFTWARE. │ +╚─────────────────────────────────────────────────────────────────────────────*/ +#include "libc/macros.internal.h" +.text.windows + +// Used by clone() on Windows to launch thread. +// +// Windows owns the stack memory when we initially enter threads. +// This function switches us over, so that we can start using the +// runtime facilities. +// +// @param %rdi is arg +// @param %rsi is func +// @param %rdx is stack +// @return %rax is exit code +// @see clone() +WinThreadLaunch: + push %rbx + push %r15 + mov %rbp,%r15 + mov %rsp,%rbx + mov %rdx,%rsp + xor %rbp,%rbp + call *%rsi + mov %r15,%rbp + mov %rbx,%rsp + pop %r15 + pop %rbx + ret + .endfn WinThreadLaunch,globl,hidden diff --git a/libc/stdio/unlocked/stdio_unlock.S b/libc/stdio/unlocked/stdio_unlock.S index 49f11ce44..dd25128a2 100644 --- a/libc/stdio/unlocked/stdio_unlock.S +++ b/libc/stdio/unlocked/stdio_unlock.S @@ -38,6 +38,7 @@ stdio_unlock: push %rbp mov %rsp,%rbp + .profilable // acquires mutex push %rcx diff --git a/libc/str/str.mk b/libc/str/str.mk index 8b2a34226..813af6569 100644 --- a/libc/str/str.mk +++ b/libc/str/str.mk @@ -86,6 +86,14 @@ o/$(MODE)/libc/str/windowstimetotimespec.o: \ OVERRIDE_CFLAGS += \ -O2 +# we can't use compiler magic because: +# kprintf() depends on these functions +o/$(MODE)/libc/fmt/strsignal.greg.o: \ + OVERRIDE_CFLAGS += \ + -fpie \ + -ffreestanding \ + $(NO_MAGIC) + LIBC_STR_LIBS = $(foreach x,$(LIBC_STR_ARTIFACTS),$($(x))) LIBC_STR_SRCS = $(foreach x,$(LIBC_STR_ARTIFACTS),$($(x)_SRCS)) LIBC_STR_HDRS = $(foreach x,$(LIBC_STR_ARTIFACTS),$($(x)_HDRS)) diff --git a/libc/str/strsignal.greg.c b/libc/str/strsignal.greg.c index a349ac47c..b169c1264 100644 --- a/libc/str/strsignal.greg.c +++ b/libc/str/strsignal.greg.c @@ -35,7 +35,9 @@ static char g_strsignal[12]; * @return pointer to static memory that mutates on subsequent calls * @see sigaction() */ -noasan noinstrument char *strsignal(int sig) { +privileged char *strsignal(int sig) { + // we need privileged because: + // kprintf is privileged and it depends on this char *p; const char *s; p = g_strsignal; diff --git a/libc/testlib/quota.c b/libc/testlib/quota.c index 815e7e994..a9e452a66 100644 --- a/libc/testlib/quota.c +++ b/libc/testlib/quota.c @@ -32,7 +32,7 @@ #include "libc/testlib/testlib.h" #include "third_party/dlmalloc/dlmalloc.h" -static noasan relegated uint64_t CountMappedBytes(void) { +static noasan noubsan relegated uint64_t CountMappedBytes(void) { size_t i; uint64_t x, y; for (x = i = 0; i < _mmi.i; ++i) { @@ -77,7 +77,7 @@ relegated void __oom_hook(size_t request) { kprintf("FIX CODE OR TUNE QUOTA += -M%dm\n", newlim / (1024 * 1024)); } kprintf("\n"); - PrintMemoryIntervals(2, &_mmi); + __print_maps(); kprintf("\nTHE STRAW THAT BROKE THE CAMEL'S BACK\n"); PrintBacktraceUsingSymbols(2, 0, GetSymbolTable()); PrintSystemMappings(2); diff --git a/test/libc/calls/sigaction_test.c b/test/libc/calls/sigaction_test.c index 2cece08e6..4f09954e0 100644 --- a/test/libc/calls/sigaction_test.c +++ b/test/libc/calls/sigaction_test.c @@ -35,7 +35,7 @@ struct sigaction oldsa; volatile bool gotsigint; void OnSigInt(int sig) { - _checkstackalign(); + CheckStackIsAligned(); gotsigint = true; } @@ -109,7 +109,7 @@ TEST(sigaction, testPingPongParentChildWithSigint) { volatile int trapeax; void OnTrap(int sig, struct siginfo *si, struct ucontext *ctx) { - _checkstackalign(); + CheckStackIsAligned(); trapeax = ctx->uc_mcontext.rax; } @@ -126,7 +126,7 @@ TEST(sigaction, debugBreak_handlerCanReadCpuState) { // test signal handler can modify cpu registers (now it's recoverable!) void SkipOverFaultingInstruction(struct ucontext *ctx) { - _checkstackalign(); + CheckStackIsAligned(); struct XedDecodedInst xedd; xed_decoded_inst_zero_set_mode(&xedd, XED_MACHINE_MODE_LONG_64); xed_instruction_length_decode(&xedd, (void *)ctx->uc_mcontext.rip, 15); @@ -134,7 +134,7 @@ void SkipOverFaultingInstruction(struct ucontext *ctx) { } void OnFpe(int sig, struct siginfo *si, struct ucontext *ctx) { - _checkstackalign(); + CheckStackIsAligned(); SkipOverFaultingInstruction(ctx); ctx->uc_mcontext.rax = 42; ctx->uc_mcontext.rdx = 0; diff --git a/test/libc/log/backtrace_test.c b/test/libc/log/backtrace_test.c index e283d19cc..9c87717bc 100644 --- a/test/libc/log/backtrace_test.c +++ b/test/libc/log/backtrace_test.c @@ -702,6 +702,7 @@ TEST(ShowCrashReports, testNpeCrashAfterFinalize) { close(fds[0]); ASSERT_NE(-1, wait(&ws)); EXPECT_TRUE(WIFEXITED(ws)); + EXPECT_EQ(0, WTERMSIG(ws)); EXPECT_EQ(IsAsan() ? 77 : 128 + SIGSEGV, WEXITSTATUS(ws)); /* NULL is stopgap until we can copy symbol tables into binary */ if (!strstr(output, IsAsan() ? "null pointer" : "Uncaught SIGSEGV (SEGV_")) { diff --git a/test/libc/rand/rand64_test.c b/test/libc/rand/rand64_test.c index 7311543a4..f57b7d516 100644 --- a/test/libc/rand/rand64_test.c +++ b/test/libc/rand/rand64_test.c @@ -38,9 +38,8 @@ #include "libc/time/time.h" #define THREADS 8 -#define ENTRIES 256 +#define ENTRIES 1024 -char locks[THREADS]; _Atomic(bool) ready; volatile uint64_t A[THREADS * ENTRIES]; @@ -49,7 +48,7 @@ void OnChld(int sig) { } dontinline void Pause(void) { - __builtin_ia32_pause(); + // when ftrace is enabled } dontinline void Generate(int i) { @@ -62,7 +61,6 @@ int Thrasher(void *arg) { for (i = 0; i < ENTRIES; ++i) { Generate(id * ENTRIES + i); } - _spunlock(locks + id); return 0; } @@ -93,24 +91,22 @@ TEST(rand64, testThreadSafety_doesntProduceIdenticalValues) { sigemptyset(&ss); sigaddset(&ss, SIGCHLD); EXPECT_EQ(0, sigprocmask(SIG_BLOCK, &ss, &oldss)); - for (i = 0; i < THREADS; ++i) { - locks[i] = 1; - } ready = false; + memset(tid, -1, sizeof(tid)); for (i = 0; i < THREADS; ++i) { - tls[i] = calloc(1, 64); - __initialize_tls(tls[i]); + tls[i] = __initialize_tls(calloc(1, 64)); stacks[i] = mmap(0, GetStackSize(), PROT_READ | PROT_WRITE, MAP_STACK | MAP_ANONYMOUS, -1, 0); - tid[i] = - clone(Thrasher, stacks[i], GetStackSize(), - CLONE_THREAD | CLONE_VM | CLONE_FS | CLONE_FILES | CLONE_SIGHAND, - (void *)(intptr_t)i, 0, tls[i], 64, 0); - ASSERT_NE(-1, tid[i]); + ASSERT_NE(-1, clone(Thrasher, stacks[i], GetStackSize(), + CLONE_THREAD | CLONE_VM | CLONE_FS | CLONE_FILES | + CLONE_SIGHAND | CLONE_SETTLS | CLONE_CHILD_CLEARTID, + (void *)(intptr_t)i, 0, tls[i], 64, tid + i)); } ready = true; for (i = 0; i < THREADS; ++i) { - _spinlock(locks + i); + _spinlock(tid + i); + EXPECT_SYS(0, 0, munmap(stacks[i], GetStackSize())); + free(tls[i]); } sigaction(SIGCHLD, &oldsa, 0); sigprocmask(SIG_BLOCK, &oldss, 0); @@ -121,10 +117,4 @@ TEST(rand64, testThreadSafety_doesntProduceIdenticalValues) { EXPECT_NE(A[i], A[j], "i=%d j=%d", i, j); } } - for (i = 0; i < THREADS; ++i) { - EXPECT_SYS(0, 0, munmap(stacks[i], GetStackSize())); - } - for (i = 0; i < THREADS; ++i) { - free(tls[i]); - } } diff --git a/test/libc/runtime/clone_test.c b/test/libc/runtime/clone_test.c index 4f4c4c409..232d5a48a 100644 --- a/test/libc/runtime/clone_test.c +++ b/test/libc/runtime/clone_test.c @@ -21,10 +21,12 @@ #include "libc/errno.h" #include "libc/intrin/kprintf.h" #include "libc/intrin/spinlock.h" +#include "libc/log/backtrace.internal.h" #include "libc/mem/mem.h" #include "libc/nexgen32e/nexgen32e.h" #include "libc/nexgen32e/threaded.h" #include "libc/runtime/stack.h" +#include "libc/runtime/symbols.internal.h" #include "libc/sysv/consts/clone.h" #include "libc/sysv/consts/map.h" #include "libc/sysv/consts/prot.h" @@ -34,14 +36,11 @@ #include "libc/time/time.h" char *stack, *tls; -int x, me, tid, thechilde; -_Alignas(64) volatile char lock; +int x, me, tid, thechilde, childetid; void SetUp(void) { x = 0; - lock = 0; me = gettid(); - thechilde = 0; tls = calloc(1, 64); __initialize_tls(tls); *(int *)(tls + 0x3c) = 31337; @@ -50,11 +49,26 @@ void SetUp(void) { } void TearDown(void) { + EXPECT_SYS(0, 0, munmap(stack, GetStackSize())); free(tls); } +int DoNothing(void *arg) { + CheckStackIsAligned(); + return 0; +} + int CloneTest1(void *arg) { - _checkstackalign(); + intptr_t rsp, top, bot; + CheckStackIsAligned(); + rsp = (intptr_t)__builtin_frame_address(0); + bot = (intptr_t)stack; + top = bot + GetStackSize(); + ASSERT_GT(rsp, bot); // check we're on stack + ASSERT_LT(rsp, top); // check we're on stack + ASSERT_GT(rsp, top - 256); // check we're near top of stack + ASSERT_TRUE(IS2POW(GetStackSize())); + ASSERT_EQ(0, bot & (GetStackSize() - 1)); x = 42; if (!IsWindows()) { ASSERT_EQ(31337, errno); @@ -65,22 +79,21 @@ int CloneTest1(void *arg) { ASSERT_EQ(23, (intptr_t)arg); thechilde = gettid(); ASSERT_NE(gettid(), getpid()); - _spunlock(&lock); - return 0; -} - -int DoNothing(void *arg) { - _checkstackalign(); + ASSERT_EQ(gettid(), childetid); // CLONE_CHILD_SETTID return 0; } TEST(clone, test1) { - _spinlock(&lock); + int ptid = 0; + _seizelock(&childetid); ASSERT_NE(-1, (tid = clone(CloneTest1, stack, GetStackSize(), CLONE_THREAD | CLONE_VM | CLONE_FS | CLONE_FILES | - CLONE_SIGHAND | CLONE_SETTLS, - (void *)23, 0, tls, 64, 0))); - _spinlock(&lock); + CLONE_SIGHAND | CLONE_PARENT_SETTID | + CLONE_CHILD_SETTID | CLONE_CHILD_CLEARTID | + CLONE_SETTLS, + (void *)23, &ptid, tls, 64, &childetid))); + _spinlock(&childetid); // CLONE_CHILD_CLEARTID + ASSERT_EQ(tid, ptid); ASSERT_EQ(42, x); ASSERT_NE(me, tid); ASSERT_EQ(tid, thechilde); @@ -91,24 +104,24 @@ TEST(clone, test1) { } int CloneTestSys(void *arg) { - _checkstackalign(); + CheckStackIsAligned(); thechilde = gettid(); ASSERT_EQ(31337, errno); open(0, 0); ASSERT_EQ(EFAULT, errno); - _spunlock(&lock); return 0; } TEST(clone, tlsSystemCallsErrno_wontClobberMainThreadBecauseTls) { ASSERT_EQ(0, errno); ASSERT_EQ(31337, *(int *)(tls + 0x3c)); - _spinlock(&lock); + _seizelock(&childetid); ASSERT_NE(-1, (tid = clone(CloneTestSys, stack, GetStackSize(), CLONE_THREAD | CLONE_VM | CLONE_FS | CLONE_FILES | - CLONE_SIGHAND | CLONE_SETTLS, - (void *)23, 0, tls, 64, 0))); - _spinlock(&lock); + CLONE_SIGHAND | CLONE_CHILD_SETTID | + CLONE_CHILD_CLEARTID | CLONE_SETTLS, + (void *)23, 0, tls, 64, &childetid))); + _spinlock(&childetid); // CLONE_CHILD_CLEARTID ASSERT_EQ(0, errno); ASSERT_EQ(EFAULT, *(int *)(tls + 0x3c)); } diff --git a/test/libc/runtime/mmap_test.c b/test/libc/runtime/mmap_test.c index 046eba15d..285581aa2 100644 --- a/test/libc/runtime/mmap_test.c +++ b/test/libc/runtime/mmap_test.c @@ -22,8 +22,8 @@ #include "libc/calls/calls.h" #include "libc/calls/ucontext.h" #include "libc/dce.h" +#include "libc/errno.h" #include "libc/fmt/fmt.h" -#include "libc/intrin/kprintf.h" #include "libc/linux/mmap.h" #include "libc/linux/munmap.h" #include "libc/log/log.h" @@ -47,6 +47,42 @@ char testlib_enable_tmp_setup_teardown; +TEST(mmap, zeroSize) { + ASSERT_SYS(EINVAL, MAP_FAILED, + mmap(NULL, 0, PROT_READ, MAP_ANONYMOUS | MAP_PRIVATE, -1, 0)); +} + +TEST(mmap, overflow) { + ASSERT_SYS(EINVAL, MAP_FAILED, + mmap(NULL, 0x800000000000, PROT_READ, MAP_ANONYMOUS | MAP_PRIVATE, + -1, 0)); + ASSERT_SYS(EINVAL, MAP_FAILED, + mmap(NULL, 0x7fffffffffff, PROT_READ, MAP_ANONYMOUS | MAP_PRIVATE, + -1, 0)); +} + +TEST(mmap, outOfAutomapRange) { + ASSERT_SYS( + ENOMEM, MAP_FAILED, + mmap(NULL, kAutomapSize, PROT_READ, MAP_ANONYMOUS | MAP_PRIVATE, -1, 0)); +} + +TEST(mmap, noreplaceImage) { + ASSERT_SYS(EEXIST, MAP_FAILED, + mmap(_base, FRAMESIZE, PROT_READ, + MAP_ANONYMOUS | MAP_PRIVATE | MAP_FIXED_NOREPLACE, -1, 0)); +} + +TEST(mmap, noreplaceExistingMap) { + char *p; + ASSERT_NE(MAP_FAILED, (p = mmap(0, FRAMESIZE, PROT_READ, + MAP_ANONYMOUS | MAP_PRIVATE, -1, 0))); + ASSERT_SYS(EEXIST, MAP_FAILED, + mmap(p, FRAMESIZE, PROT_READ, + MAP_ANONYMOUS | MAP_PRIVATE | MAP_FIXED_NOREPLACE, -1, 0)); + EXPECT_SYS(0, 0, munmap(p, FRAMESIZE)); +} + TEST(mmap, testMapFile) { int fd; char *p; @@ -148,6 +184,25 @@ TEST(mmap, mapPrivate_writesDontChangeFile) { EXPECT_NE(-1, close(fd)); } +TEST(mmap, twoPowerSize_automapsAddressWithThatAlignment) { + char *q, *p; + // increase the likelihood automap is unaligned w.r.t. following call + ASSERT_NE(MAP_FAILED, (q = mmap(NULL, 0x00010000, PROT_READ | PROT_WRITE, + MAP_SHARED | MAP_ANONYMOUS, -1, 0))); + // ask for a nice big round size + ASSERT_NE(MAP_FAILED, (p = mmap(NULL, 0x00080000, PROT_READ | PROT_WRITE, + MAP_SHARED | MAP_ANONYMOUS, -1, 0))); + // verify it's aligned + ASSERT_EQ(0, (intptr_t)p & 0x0007ffff); + EXPECT_SYS(0, 0, munmap(p, 0x00080000)); + // now try again with a big size that isn't a two power + ASSERT_NE(MAP_FAILED, (p = mmap(NULL, 0x00070000, PROT_READ | PROT_WRITE, + MAP_SHARED | MAP_ANONYMOUS, -1, 0))); + // automap doesn't bother aligning it + ASSERT_NE(0, (intptr_t)p & 0x0007ffff); + EXPECT_SYS(0, 0, munmap(q, 0x00010000)); +} + TEST(isheap, nullPtr) { ASSERT_FALSE(_isheap(NULL)); } diff --git a/test/libc/runtime/munmap_test.c b/test/libc/runtime/munmap_test.c index 47b230356..3cbbafa68 100644 --- a/test/libc/runtime/munmap_test.c +++ b/test/libc/runtime/munmap_test.c @@ -233,12 +233,12 @@ TEST(munmap, tinyFile_preciseUnmapSize) { // clang-format off TEST(munmap, tinyFile_mapThriceUnmapOnce) { - char *p; + char *p = (char *)0x02000000; ASSERT_SYS(0, 3, open("doge", O_RDWR | O_CREAT | O_TRUNC, 0644)); ASSERT_SYS (0, 5, write(3, "hello", 5)); - ASSERT_NE(MAP_FAILED, (p=mmap(0, FRAMESIZE, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0))); - ASSERT_NE(MAP_FAILED, mmap(p+FRAMESIZE*1, 5, PROT_READ, MAP_PRIVATE, 3, 0)); - ASSERT_NE(MAP_FAILED, mmap(p+FRAMESIZE*3, 5, PROT_READ, MAP_PRIVATE, 3, 0)); + ASSERT_NE(MAP_FAILED, mmap(p+FRAMESIZE*0, FRAMESIZE, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS|MAP_FIXED, -1, 0)); + ASSERT_NE(MAP_FAILED, mmap(p+FRAMESIZE*1, 5, PROT_READ, MAP_PRIVATE|MAP_FIXED, 3, 0)); + ASSERT_NE(MAP_FAILED, mmap(p+FRAMESIZE*3, 5, PROT_READ, MAP_PRIVATE|MAP_FIXED, 3, 0)); ASSERT_SYS(0, 0, close(3)); EXPECT_TRUE(MemoryExists(p+FRAMESIZE*0)); EXPECT_TRUE(MemoryExists(p+FRAMESIZE*1)); diff --git a/third_party/dlmalloc/dlmalloc.mk b/third_party/dlmalloc/dlmalloc.mk index d2cc826bb..6c009af2c 100644 --- a/third_party/dlmalloc/dlmalloc.mk +++ b/third_party/dlmalloc/dlmalloc.mk @@ -58,7 +58,7 @@ o/$(MODE)/third_party/dlmalloc/dlmalloc.o: \ # we must segregate codegen because: # file contains multiple independently linkable apis -o/$(MODE)/third_party/dlmalloc/dlmalloc.greg.o: \ +o/$(MODE)/third_party/dlmalloc/dlmalloc.o: \ OVERRIDE_CFLAGS += \ -ffunction-sections \ -fdata-sections diff --git a/third_party/dlmalloc/vespene.greg.c b/third_party/dlmalloc/vespene.c similarity index 100% rename from third_party/dlmalloc/vespene.greg.c rename to third_party/dlmalloc/vespene.c diff --git a/third_party/lua/lrepl.c b/third_party/lua/lrepl.c index afa16db70..33eb69ac4 100644 --- a/third_party/lua/lrepl.c +++ b/third_party/lua/lrepl.c @@ -255,7 +255,7 @@ static ssize_t pushline (lua_State *L, int firstline) { rc = b ? 1 : -1; } if (!(rc == -1 && errno == EAGAIN)) { - write(1, "\n", 1); + write(1, "\n", 1); } if (rc == -1 || (!rc && !b)) { return rc; diff --git a/tool/build/runit.c b/tool/build/runit.c index 63109d038..b11bce9c7 100644 --- a/tool/build/runit.c +++ b/tool/build/runit.c @@ -314,8 +314,8 @@ static void Send(const void *output, size_t outputsize) { static z_stream zs; static char zbuf[4096]; if (!once) { - CHECK_EQ(Z_OK, deflateInit2(&zs, Z_DEFAULT_COMPRESSION, Z_DEFLATED, - MAX_WBITS, DEF_MEM_LEVEL, Z_DEFAULT_STRATEGY)); + CHECK_EQ(Z_OK, deflateInit2(&zs, 4, Z_DEFLATED, MAX_WBITS, DEF_MEM_LEVEL, + Z_DEFAULT_STRATEGY)); once = true; } zs.next_in = output; diff --git a/tool/net/redbean.c b/tool/net/redbean.c index 8bf8de2ab..45ab5e634 100644 --- a/tool/net/redbean.c +++ b/tool/net/redbean.c @@ -60,6 +60,7 @@ #include "libc/nexgen32e/nt2sysv.h" #include "libc/nexgen32e/rdtsc.h" #include "libc/nexgen32e/rdtscp.h" +#include "libc/nexgen32e/threaded.h" #include "libc/nt/enum/fileflagandattributes.h" #include "libc/nt/runtime.h" #include "libc/nt/thread.h" @@ -406,7 +407,6 @@ static bool sslcliused; static bool loglatency; static bool terminated; static bool uniprocess; -static bool memmonalive; static bool invalidated; static bool logmessages; static bool isinitialized; @@ -430,7 +430,7 @@ static bool loggednetworkorigin; static bool ishandlingconnection; static bool hasonclientconnection; static bool evadedragnetsurveillance; -static _Atomic(bool) terminatemonitor; +_Atomic(bool) static terminatemonitor; static int zfd; static int frags; @@ -440,6 +440,7 @@ static int mainpid; static int sandboxed; static int changeuid; static int changegid; +static int monitortid; static int isyielding; static int statuscode; static int shutdownsig; @@ -6391,7 +6392,7 @@ static int ExitWorker(void) { } if (monitortty) { terminatemonitor = true; - _spinlock(&memmonalive); + _spinlock(&monitortid); } _Exit(0); } @@ -6505,7 +6506,6 @@ static int MemoryMonitor(void *arg) { struct MemoryInterval *mi, *mi2; long i, j, k, n, x, y, pi, gen, pages; int rc, id, color, color2, workers; - _spinlock(&memmonalive); id = atomic_load_explicit(&shared->workers, memory_order_relaxed); DEBUGF("(memv) started for pid %d on tid %d", getpid(), gettid()); @@ -6538,8 +6538,7 @@ static int MemoryMonitor(void *arg) { _spunlock(&shared->montermlock); if (tty != -1) { - for (gen = 0, mi = 0, b = 0;;) { - if (terminatemonitor) break; + for (gen = 0, mi = 0, b = 0; !terminatemonitor;) { workers = atomic_load_explicit(&shared->workers, memory_order_relaxed); if (id) id = MAX(1, MIN(id, workers)); if (!id && workers) { @@ -6566,7 +6565,7 @@ static int MemoryMonitor(void *arg) { } _spunlock(&_mmi.lock); if (!ok) { - WARNF("(memv) retrying due to contention on mmap table"); + VERBOSEF("(memv) retrying due to contention on mmap table"); continue; } @@ -6632,18 +6631,25 @@ static int MemoryMonitor(void *arg) { free(mi); free(b); } - _spunlock(&memmonalive); + DEBUGF("(memv) done"); return 0; } -static int MonitorMemory(void) { - return clone(MemoryMonitor, - mmap(0, FRAMESIZE, PROT_READ | PROT_WRITE, - MAP_STACK | MAP_ANONYMOUS, -1, 0), - FRAMESIZE, - CLONE_VM | CLONE_THREAD | CLONE_FS | CLONE_FILES | CLONE_SIGHAND, - 0, 0, 0, 0, 0); +static void MonitorMemory(void) { + char *tls; + monitortid = -1; + tls = __initialize_tls(malloc(64)); + __cxa_atexit(free, tls, 0); + if (clone(MemoryMonitor, + mmap(0, GetStackSize(), PROT_READ | PROT_WRITE, + MAP_STACK | MAP_ANONYMOUS, -1, 0), + GetStackSize(), + CLONE_VM | CLONE_THREAD | CLONE_FS | CLONE_FILES | CLONE_SIGHAND | + CLONE_SETTLS | CLONE_CHILD_SETTID | CLONE_CHILD_CLEARTID, + 0, 0, tls, 64, &monitortid) == -1) { + monitortid = 0; + } } static int HandleConnection(size_t i) { @@ -6665,7 +6671,6 @@ static int HandleConnection(size_t i) { switch ((pid = fork())) { case 0: if (monitortty) { - memmonalive = false; MonitorMemory(); } meltdown = false; @@ -6820,8 +6825,9 @@ static int HandleReadline(void) { errno = 0; return 0; } else { - OnTerm(SIGIO); // error - return -1; + WARNF("unexpected terminal error %d% m", status); + errno = 0; + return 0; } } linenoiseDisableRawMode(); @@ -7309,7 +7315,7 @@ void RedBean(int argc, char *argv[]) { } if (monitortty) { terminatemonitor = true; - _spinlock(&memmonalive); + _spinlock(&monitortid); } INFOF("(srvr) shutdown complete"); }