From fec396037ac84f915dccd84d7123efafdfaebf2c Mon Sep 17 00:00:00 2001 From: Justine Tunney Date: Thu, 12 May 2022 17:52:13 -0700 Subject: [PATCH] Get threads working on all platforms We now have a high-quality clone() implementation for creating lightweight threads on Linux/Windows/FreeBSD/NetBSD/OpenBSD. --- examples/shell.c | 35 +- libc/calls/calls.h | 8 +- libc/calls/internal.h | 3 + libc/calls/mremap-sysv.greg.c | 4 +- libc/calls/tgkill.c | 34 ++ libc/calls/tkill.c | 61 +++ libc/intrin/exit1.greg.c | 17 +- libc/intrin/gettid.greg.c | 17 +- libc/intrin/kprintf.greg.c | 2 +- libc/intrin/setjmp.internal.h | 29 ++ libc/intrin/spinlock.h | 40 +- libc/intrin/threaded.c | 1 + libc/intrin/threaded.internal.h | 10 + libc/intrin/winthread.internal.h | 7 +- libc/log/showcrashreports.c | 8 +- libc/nexgen32e/nt2sysv.S | 2 +- libc/nt/kernel32/TerminateThread.s | 10 + libc/nt/master.sh | 2 +- libc/runtime/arch_prctl.c | 2 +- libc/runtime/clone.c | 535 +++++++++++++++++++++ libc/runtime/fork-nt.c | 4 +- libc/runtime/mmap.c | 29 +- libc/sysv/calls/__tfork.s | 2 - libc/sysv/calls/getcontext.s | 2 - libc/sysv/calls/sys_getcontext.s | 2 + libc/sysv/calls/sys_tgkill.s | 2 + libc/sysv/calls/sys_tkill.s | 2 + libc/sysv/calls/tgkill.s | 2 - libc/sysv/calls/tkill.s | 2 - libc/sysv/consts.sh | 4 +- libc/sysv/consts/__NR_bsdthread_create.S | 2 - libc/sysv/consts/__NR_bsdthread_register.S | 2 - libc/sysv/consts/nr.h | 4 - libc/sysv/syscalls.sh | 7 +- libc/thread/clone.c | 400 --------------- libc/thread/freebsd.internal.h | 11 - libc/thread/openbsd.internal.h | 14 - libc/thread/thread.mk | 5 +- libc/thread/xnu.internal.h | 14 +- test/libc/calls/tkill_test.c | 0 test/libc/intrin/gettid_test.c | 0 test/libc/rand/rand64_test.c | 14 +- test/libc/{thread => runtime}/clone_test.c | 31 +- 43 files changed, 850 insertions(+), 532 deletions(-) create mode 100644 libc/calls/tgkill.c create mode 100644 libc/calls/tkill.c create mode 100644 libc/intrin/setjmp.internal.h create mode 100644 libc/intrin/threaded.internal.h create mode 100644 libc/runtime/clone.c delete mode 100644 libc/sysv/calls/__tfork.s delete mode 100644 libc/sysv/calls/getcontext.s create mode 100644 libc/sysv/calls/sys_getcontext.s create mode 100644 libc/sysv/calls/sys_tgkill.s create mode 100644 libc/sysv/calls/sys_tkill.s delete mode 100644 libc/sysv/calls/tgkill.s delete mode 100644 libc/sysv/calls/tkill.s delete mode 100644 libc/sysv/consts/__NR_bsdthread_create.S delete mode 100644 libc/sysv/consts/__NR_bsdthread_register.S delete mode 100644 libc/thread/clone.c delete mode 100644 libc/thread/openbsd.internal.h create mode 100644 test/libc/calls/tkill_test.c create mode 100644 test/libc/intrin/gettid_test.c rename test/libc/{thread => runtime}/clone_test.c (80%) diff --git a/examples/shell.c b/examples/shell.c index 117ea81e7..19c9f2082 100644 --- a/examples/shell.c +++ b/examples/shell.c @@ -11,16 +11,21 @@ #include "libc/calls/sigbits.h" #include "libc/calls/struct/sigaction.h" #include "libc/calls/struct/sigset.h" +#include "libc/calls/struct/timespec.h" #include "libc/fmt/fmt.h" #include "libc/fmt/itoa.h" #include "libc/log/internal.h" +#include "libc/log/log.h" #include "libc/macros.internal.h" #include "libc/runtime/internal.h" #include "libc/runtime/runtime.h" +#include "libc/stdio/append.internal.h" #include "libc/stdio/stdio.h" #include "libc/str/str.h" +#include "libc/sysv/consts/clock.h" #include "libc/sysv/consts/dt.h" #include "libc/sysv/consts/sig.h" +#include "libc/time/time.h" #include "libc/x/x.h" #include "third_party/linenoise/linenoise.h" @@ -102,7 +107,11 @@ static char *ShellHint(const char *p, const char **ansi1, const char **ansi2) { } int main(int argc, char *argv[]) { + bool timeit; + int64_t nanos; int n, ws, pid; + struct rusage ru; + struct timespec ts1, ts2; char *prog, path[PATH_MAX]; sigset_t chldmask, savemask; struct sigaction ignore, saveint, savequit; @@ -114,6 +123,12 @@ int main(int argc, char *argv[]) { while ((line = linenoiseWithHistory(prompt, "cmd"))) { n = 0; start = line; + if (startswith(start, "time ")) { + timeit = true; + start += 5; + } else { + timeit = false; + } args = xcalloc(1, sizeof(*args)); while ((arg = strtok_r(start, " \t\r\n", &state))) { args = xrealloc(args, (++n + 1) * sizeof(*args)); @@ -132,6 +147,9 @@ int main(int argc, char *argv[]) { sigaddset(&chldmask, SIGCHLD); sigprocmask(SIG_BLOCK, &chldmask, &savemask); + if (timeit) { + clock_gettime(CLOCK_REALTIME, &ts1); + } if (!fork()) { sigaction(SIGINT, &saveint, 0); sigaction(SIGQUIT, &savequit, 0); @@ -139,8 +157,23 @@ int main(int argc, char *argv[]) { execv(prog, args); _Exit(127); } + wait4(0, &ws, 0, &ru); + if (timeit) { + clock_gettime(CLOCK_REALTIME, &ts2); + if (ts2.tv_sec == ts1.tv_sec) { + nanos = ts2.tv_nsec - ts1.tv_nsec; + } else { + nanos = (ts2.tv_sec - ts1.tv_sec) * 1000000000LL; + nanos += 1000000000LL - ts1.tv_nsec; + nanos += ts2.tv_nsec; + } + printf("took %,ldµs wall time\n", nanos / 1000); + p = 0; + AppendResourceReport(&p, &ru, "\n"); + fputs(p, stdout); + free(p); + } - wait(&ws); p = prompt; if (WIFEXITED(ws)) { if (WEXITSTATUS(ws)) { diff --git a/libc/calls/calls.h b/libc/calls/calls.h index ee64d4a59..afb1060c5 100644 --- a/libc/calls/calls.h +++ b/libc/calls/calls.h @@ -133,6 +133,8 @@ int getpgrp(void) nosideeffect; int getpid(void); int getppid(void); int getpriority(int, unsigned); +int getresgid(uint32_t *, uint32_t *, uint32_t *); +int getresuid(uint32_t *, uint32_t *, uint32_t *); int getrlimit(int, struct rlimit *); int getrusage(int, struct rusage *); int getsid(int) nosideeffect; @@ -163,6 +165,7 @@ int pause(void); int personality(uint64_t); int pipe(int[hasatleast 2]); int pipe2(int[hasatleast 2], int); +int pledge(const char *, const char *); int posix_fadvise(int, uint64_t, uint64_t, int); int posix_madvise(void *, uint64_t, int); int prctl(int, ...); @@ -186,8 +189,6 @@ int setpriority(int, unsigned, int); int setregid(uint32_t, uint32_t); int setresgid(uint32_t, uint32_t, uint32_t); int setresuid(uint32_t, uint32_t, uint32_t); -int getresgid(uint32_t *, uint32_t *, uint32_t *); -int getresuid(uint32_t *, uint32_t *, uint32_t *); int setreuid(uint32_t, uint32_t); int setrlimit(int, const struct rlimit *); int setsid(void); @@ -203,6 +204,8 @@ int symlinkat(const char *, int, const char *); int sync_file_range(int, int64_t, int64_t, unsigned); int sysctl(const int *, unsigned, void *, size_t *, void *, size_t); int sysinfo(struct sysinfo *); +int tgkill(int, int, int); +int tkill(int, int); int touch(const char *, uint32_t); int truncate(const char *, uint64_t); int ttyname_r(int, char *, size_t); @@ -239,7 +242,6 @@ ssize_t write(int, const void *, size_t); struct dirent *readdir(DIR *); void rewinddir(DIR *); void sync(void); -int pledge(const char *, const char *); int clone(int (*)(void *), void *, size_t, int, void *, int *, void *, size_t, int *); diff --git a/libc/calls/internal.h b/libc/calls/internal.h index 286b1cd1a..c50bba12d 100644 --- a/libc/calls/internal.h +++ b/libc/calls/internal.h @@ -158,6 +158,7 @@ i32 sys_fsync(i32) hidden; i32 sys_ftruncate(i32, i64, i64) hidden; i32 sys_futimes(i32, const struct timeval *) hidden; i32 sys_futimesat(i32, const char *, const struct timeval *) hidden; +i32 sys_getcontext(void *) hidden; i32 sys_getitimer(i32, struct itimerval *) hidden; i32 sys_getpgid(i32) hidden; i32 sys_getpgrp(void) hidden; @@ -212,6 +213,8 @@ i32 sys_symlinkat(const char *, i32, const char *) hidden; i32 sys_sync(void) hidden; i32 sys_sync_file_range(i32, i64, i64, u32) hidden; i32 sys_sysinfo(struct sysinfo *) hidden; +i32 sys_tgkill(i32, i32, i32) hidden; +i32 sys_tkill(i32, i32, void *) hidden; i32 sys_truncate(const char *, u64, u64) hidden; i32 sys_uname(char *) hidden; i32 sys_unlinkat(i32, const char *, i32) hidden; diff --git a/libc/calls/mremap-sysv.greg.c b/libc/calls/mremap-sysv.greg.c index c44cff2d1..79b1c40bc 100644 --- a/libc/calls/mremap-sysv.greg.c +++ b/libc/calls/mremap-sysv.greg.c @@ -50,8 +50,8 @@ privileged void *sys_mremap(void *p, size_t n, size_t m, int f, void *q) { r10 = m; r8 = (f & MREMAP_FIXED) ? MAP_FIXED : 0; asm(CFLAG_ASM("syscall") - : CFLAG_CONSTRAINT(cf), "+a"(rax) - : "D"(p), "S"(n), "d"(q), "r"(r10), "r"(r8) + : CFLAG_CONSTRAINT(cf), "+a"(rax), "=d"(rdx) + : "D"(p), "S"(n), "2"(q), "r"(r10), "r"(r8) : "rcx", "r9", "r11", "memory", "cc"); if (cf) errno = rax, rax = -1; } else { diff --git a/libc/calls/tgkill.c b/libc/calls/tgkill.c new file mode 100644 index 000000000..a729b4c6d --- /dev/null +++ b/libc/calls/tgkill.c @@ -0,0 +1,34 @@ +/*-*- mode:c;indent-tabs-mode:nil;c-basic-offset:2;tab-width:8;coding:utf-8 -*-│ +│vi: set net ft=c ts=2 sts=2 sw=2 fenc=utf-8 :vi│ +╞══════════════════════════════════════════════════════════════════════════════╡ +│ Copyright 2022 Justine Alexandra Roberts Tunney │ +│ │ +│ Permission to use, copy, modify, and/or distribute this software for │ +│ any purpose with or without fee is hereby granted, provided that the │ +│ above copyright notice and this permission notice appear in all copies. │ +│ │ +│ THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL │ +│ WARRANTIES WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED │ +│ WARRANTIES OF MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE │ +│ AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL │ +│ DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR │ +│ PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER │ +│ TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR │ +│ PERFORMANCE OF THIS SOFTWARE. │ +╚─────────────────────────────────────────────────────────────────────────────*/ +#include "libc/calls/calls.h" +#include "libc/calls/internal.h" +#include "libc/calls/strace.internal.h" + +/** + * Kills thread group. + * + * @raises ENOSYS on non-Linux + * @see tkill() + */ +int tgkill(int tgid, int tid, int sig) { + int rc; + rc = sys_tgkill(tgid, tid, sig); + STRACE("tgkill(%d, %d, %G) → %d% m", tgid, tid, sig, rc); + return rc; +} diff --git a/libc/calls/tkill.c b/libc/calls/tkill.c new file mode 100644 index 000000000..69c9576ae --- /dev/null +++ b/libc/calls/tkill.c @@ -0,0 +1,61 @@ +/*-*- mode:c;indent-tabs-mode:nil;c-basic-offset:2;tab-width:8;coding:utf-8 -*-│ +│vi: set net ft=c ts=2 sts=2 sw=2 fenc=utf-8 :vi│ +╞══════════════════════════════════════════════════════════════════════════════╡ +│ Copyright 2022 Justine Alexandra Roberts Tunney │ +│ │ +│ Permission to use, copy, modify, and/or distribute this software for │ +│ any purpose with or without fee is hereby granted, provided that the │ +│ above copyright notice and this permission notice appear in all copies. │ +│ │ +│ THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL │ +│ WARRANTIES WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED │ +│ WARRANTIES OF MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE │ +│ AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL │ +│ DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR │ +│ PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER │ +│ TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR │ +│ PERFORMANCE OF THIS SOFTWARE. │ +╚─────────────────────────────────────────────────────────────────────────────*/ +#include "libc/calls/calls.h" +#include "libc/calls/internal.h" +#include "libc/calls/strace.internal.h" +#include "libc/dce.h" +#include "libc/intrin/kprintf.h" +#include "libc/nt/enum/threadaccess.h" +#include "libc/nt/runtime.h" +#include "libc/nt/thread.h" +#include "libc/sysv/errfuns.h" + +static textwindows int sys_tkill_nt(int tid, int sig) { + int rc; + int64_t hand; + if ((hand = OpenThread(kNtThreadTerminate, false, tid))) { + if (TerminateThread(hand, 128 + sig)) { + rc = 0; + } else { + rc = __winerr(); + } + CloseHandle(hand); + } else { + rc = esrch(); + } + return rc; +} + +/** + * Kills thread. + * + * @param tid is thread id + * @param sig does nothing on xnu + * @return 0 on success, or -1 w/ errno + */ +int tkill(int tid, int sig) { + int rc; + if (!IsWindows()) { + rc = sys_tkill(tid, sig, 0); + } else { + rc = sys_tkill_nt(tid, sig); + } + STRACE("tkill(%d, %G) → %d% m", tid, sig, rc); + return rc; +} diff --git a/libc/intrin/exit1.greg.c b/libc/intrin/exit1.greg.c index 2b20fec0e..90e43e5f1 100644 --- a/libc/intrin/exit1.greg.c +++ b/libc/intrin/exit1.greg.c @@ -16,14 +16,12 @@ │ TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR │ │ PERFORMANCE OF THIS SOFTWARE. │ ╚─────────────────────────────────────────────────────────────────────────────*/ -#include "libc/bits/weaken.h" -#include "libc/calls/internal.h" #include "libc/calls/strace.internal.h" #include "libc/dce.h" +#include "libc/intrin/setjmp.internal.h" #include "libc/intrin/winthread.internal.h" -#include "libc/mem/mem.h" -#include "libc/nt/runtime.h" #include "libc/nt/thread.h" +#include "libc/runtime/runtime.h" #include "libc/sysv/consts/nr.h" /** @@ -36,18 +34,15 @@ */ privileged wontreturn void _Exit1(int rc) { struct WinThread *wt; - STRACE("_Exit1(%d)", rc); + /* STRACE("_Exit1(%d)", rc); */ if (!IsWindows() && !IsMetal()) { + register long r10 asm("r10") = 0; asm volatile("syscall" : /* no outputs */ - : "a"(__NR_exit), "D"(IsLinux() ? rc : 0) + : "a"(__NR_exit), "D"(IsLinux() ? rc : 0), "S"(0), "d"(0), + "r"(r10) : "rcx", "r11", "memory"); - __builtin_unreachable(); } else if (IsWindows()) { - if ((wt = GetWinThread())) { - __releasefd(wt->pid); - weaken(free)(wt); - } ExitThread(rc); } for (;;) { diff --git a/libc/intrin/gettid.greg.c b/libc/intrin/gettid.greg.c index 4a8f92703..28c5cc548 100644 --- a/libc/intrin/gettid.greg.c +++ b/libc/intrin/gettid.greg.c @@ -19,18 +19,21 @@ #include "libc/calls/calls.h" #include "libc/dce.h" #include "libc/intrin/tls.h" -#include "libc/intrin/winthread.internal.h" #include "libc/nt/thread.h" /** * Returns current thread id. * @asyncsignalsafe */ -int gettid(void) { +privileged int gettid(void) { int rc; int64_t wut; struct WinThread *wt; + if (IsWindows()) { + return GetCurrentThreadId(); + } + if (IsLinux()) { asm("syscall" : "=a"(rc) // man says always succeeds @@ -59,7 +62,7 @@ int gettid(void) { asm("syscall" : "=a"(rc) // man says always succeeds : "0"(311) // _lwp_self() - : "rcx", "r11", "memory", "cc"); + : "rcx", "rdx", "r11", "memory", "cc"); return rc; } @@ -73,13 +76,5 @@ int gettid(void) { return wut; // narrowing intentional } - if (IsWindows()) { - if ((wt = GetWinThread())) { - return wt->pid; - } else { - return GetCurrentThreadId(); - } - } - return getpid(); } diff --git a/libc/intrin/kprintf.greg.c b/libc/intrin/kprintf.greg.c index cc3681873..fb1b5b60a 100644 --- a/libc/intrin/kprintf.greg.c +++ b/libc/intrin/kprintf.greg.c @@ -31,6 +31,7 @@ #include "libc/intrin/lockcmpxchg.h" #include "libc/intrin/nomultics.internal.h" #include "libc/intrin/spinlock.h" +#include "libc/intrin/threaded.internal.h" #include "libc/limits.h" #include "libc/log/internal.h" #include "libc/macros.internal.h" @@ -56,7 +57,6 @@ struct Timestamps { unsigned long long start; }; -extern bool __threaded; unsigned long long __kbirth; // see fork-nt.c privileged static struct Timestamps kenter(void) { diff --git a/libc/intrin/setjmp.internal.h b/libc/intrin/setjmp.internal.h new file mode 100644 index 000000000..b48150874 --- /dev/null +++ b/libc/intrin/setjmp.internal.h @@ -0,0 +1,29 @@ +#ifndef COSMOPOLITAN_LIBC_INTRIN_SETJMP_INTERNAL_H_ +#define COSMOPOLITAN_LIBC_INTRIN_SETJMP_INTERNAL_H_ +#include "libc/limits.h" +#if !(__ASSEMBLER__ + __LINKER__ + 0) +COSMOPOLITAN_C_START_ + +/** + * Encodes nonzero number for longjmp(). + * + * This is a workaround to the fact that the value has to be non-zero. + * So we work around it by dedicating the highest bit to being a flag. + */ +static inline int EncodeLongjmp(int x) { + return x | INT_MIN; +} + +/** + * Decodes nonzero number returned by setjmp(). + * + * This is a workaround to the fact that the value has to be non-zero. + * So we work around it by dedicating the highest bit to being a flag. + */ +static inline int DecodeSetjmp(int x) { + return (int)((unsigned)x << 1) >> 1; +} + +COSMOPOLITAN_C_END_ +#endif /* !(__ASSEMBLER__ + __LINKER__ + 0) */ +#endif /* COSMOPOLITAN_LIBC_INTRIN_SETJMP_INTERNAL_H_ */ diff --git a/libc/intrin/spinlock.h b/libc/intrin/spinlock.h index e8cdffaa2..28e6ddb01 100644 --- a/libc/intrin/spinlock.h +++ b/libc/intrin/spinlock.h @@ -2,27 +2,33 @@ #define COSMOPOLITAN_LIBC_INTRIN_SPINLOCK_H_ #ifdef TINY -#define _spinlock(lock) \ - do { \ - while (__sync_lock_test_and_set(lock, 1)) { \ - __builtin_ia32_pause(); \ - } \ +#define _spinlock(lock) \ + do { \ + while (__atomic_test_and_set(lock, __ATOMIC_SEQ_CST)) { \ + __builtin_ia32_pause(); \ + } \ } while (0) #else -#define _spinlock(lock) \ - do { \ - for (;;) { \ - typeof(*(lock)) x; \ - __atomic_load(lock, &x, __ATOMIC_RELAXED); \ - if (!x && !__sync_lock_test_and_set(lock, 1)) { \ - break; \ - } else { \ - __builtin_ia32_pause(); \ - } \ - } \ +#define _spinlock(lock) \ + do { \ + for (;;) { \ + typeof(*(lock)) x; \ + __atomic_load(lock, &x, __ATOMIC_RELAXED); \ + if (!x && !__atomic_test_and_set(lock, __ATOMIC_SEQ_CST)) { \ + break; \ + } else { \ + __builtin_ia32_pause(); \ + } \ + } \ } while (0) #endif -#define _spunlock(lock) __sync_lock_release(lock) +#define _spunlock(lock) __atomic_clear(lock, __ATOMIC_RELAXED) + +#define _seizelock(lock) \ + do { \ + typeof(*(lock)) x = 1; \ + __atomic_store(lock, &x, __ATOMIC_SEQ_CST); \ + } while (0) #endif /* COSMOPOLITAN_LIBC_INTRIN_SPINLOCK_H_ */ diff --git a/libc/intrin/threaded.c b/libc/intrin/threaded.c index 0971a44a6..c8589830b 100644 --- a/libc/intrin/threaded.c +++ b/libc/intrin/threaded.c @@ -16,5 +16,6 @@ │ TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR │ │ PERFORMANCE OF THIS SOFTWARE. │ ╚─────────────────────────────────────────────────────────────────────────────*/ +#include "libc/intrin/threaded.internal.h" bool __threaded; diff --git a/libc/intrin/threaded.internal.h b/libc/intrin/threaded.internal.h new file mode 100644 index 000000000..7b31c0b8c --- /dev/null +++ b/libc/intrin/threaded.internal.h @@ -0,0 +1,10 @@ +#ifndef COSMOPOLITAN_LIBC_INTRIN_THREADED_INTERNAL_H_ +#define COSMOPOLITAN_LIBC_INTRIN_THREADED_INTERNAL_H_ +#if !(__ASSEMBLER__ + __LINKER__ + 0) +COSMOPOLITAN_C_START_ + +extern bool __threaded; + +COSMOPOLITAN_C_END_ +#endif /* !(__ASSEMBLER__ + __LINKER__ + 0) */ +#endif /* COSMOPOLITAN_LIBC_INTRIN_THREADED_INTERNAL_H_ */ diff --git a/libc/intrin/winthread.internal.h b/libc/intrin/winthread.internal.h index 2464dbd58..0ea54dbd1 100644 --- a/libc/intrin/winthread.internal.h +++ b/libc/intrin/winthread.internal.h @@ -1,11 +1,16 @@ #ifndef COSMOPOLITAN_LIBC_RUNTIME_WINTHREAD_INTERNAL_H_ #define COSMOPOLITAN_LIBC_RUNTIME_WINTHREAD_INTERNAL_H_ #include "libc/intrin/tls.h" +#include "libc/runtime/runtime.h" #if !(__ASSEMBLER__ + __LINKER__ + 0) COSMOPOLITAN_C_START_ struct WinThread { - int pid; + uint32_t tid; + int flags; + int *ctid; + int (*func)(void *); + void *arg; }; extern int __winthread; diff --git a/libc/log/showcrashreports.c b/libc/log/showcrashreports.c index 848c51c3b..66edf6490 100644 --- a/libc/log/showcrashreports.c +++ b/libc/log/showcrashreports.c @@ -19,8 +19,6 @@ #include "libc/calls/sigbits.h" #include "libc/calls/struct/sigaction.h" #include "libc/calls/struct/sigaltstack.h" -#include "libc/intrin/kprintf.h" -#include "libc/log/backtrace.internal.h" #include "libc/log/internal.h" #include "libc/log/log.h" #include "libc/macros.internal.h" @@ -35,10 +33,6 @@ STATIC_YOINK("__get_symbol_by_addr"); /* for asan memory origin */ extern const unsigned char __oncrash_thunks[8][11]; -static void FreeSigaltstack(void *p) { - free(p); -} - /** * Installs crash signal handlers. * @@ -73,7 +67,7 @@ void ShowCrashReports(void) { ss.ss_flags = 0; ss.ss_size = SIGSTKSZ; ss.ss_sp = malloc(SIGSTKSZ); - __cxa_atexit(FreeSigaltstack, ss.ss_sp, 0); + __cxa_atexit(free, ss.ss_sp, 0); sa.sa_flags = SA_SIGINFO | SA_NODEFER | SA_ONSTACK; sigfillset(&sa.sa_mask); for (i = 0; i < ARRAYLEN(kCrashSigs); ++i) { diff --git a/libc/nexgen32e/nt2sysv.S b/libc/nexgen32e/nt2sysv.S index 7f804f1a5..ef5939d5b 100644 --- a/libc/nexgen32e/nt2sysv.S +++ b/libc/nexgen32e/nt2sysv.S @@ -37,7 +37,7 @@ __nt2sysv: push %rbx push %rdi push %rsi - pushf + pushf # TODO(jart): Do we need it? lea -0x80(%rbp),%rdi call _savexmm mov %rcx,%rdi diff --git a/libc/nt/kernel32/TerminateThread.s b/libc/nt/kernel32/TerminateThread.s index b00b0693f..1b2bf9f45 100644 --- a/libc/nt/kernel32/TerminateThread.s +++ b/libc/nt/kernel32/TerminateThread.s @@ -1,2 +1,12 @@ .include "o/libc/nt/codegen.inc" .imp kernel32,__imp_TerminateThread,TerminateThread,0 + + .text.windows +TerminateThread: + push %rbp + mov %rsp,%rbp + .profilable + mov __imp_TerminateThread(%rip),%rax + jmp __sysv2nt + .endfn TerminateThread,globl + .previous diff --git a/libc/nt/master.sh b/libc/nt/master.sh index 50db7cd24..caefb97e5 100755 --- a/libc/nt/master.sh +++ b/libc/nt/master.sh @@ -1208,7 +1208,7 @@ imp 'SystemTimeToFileTime' SystemTimeToFileTime kernel32 0 2 imp 'SystemTimeToTzSpecificLocalTime' SystemTimeToTzSpecificLocalTime kernel32 0 imp 'SystemTimeToTzSpecificLocalTimeEx' SystemTimeToTzSpecificLocalTimeEx kernel32 0 imp 'TerminateJobObject' TerminateJobObject kernel32 1426 -imp 'TerminateThread' TerminateThread kernel32 0 +imp 'TerminateThread' TerminateThread kernel32 0 2 imp 'TermsrvAppInstallMode' TermsrvAppInstallMode kernel32 1429 imp 'TermsrvConvertSysRootToUserDir' TermsrvConvertSysRootToUserDir kernel32 1430 imp 'TermsrvCreateRegEntry' TermsrvCreateRegEntry kernel32 1431 diff --git a/libc/runtime/arch_prctl.c b/libc/runtime/arch_prctl.c index b8123e1e2..c5602feb4 100644 --- a/libc/runtime/arch_prctl.c +++ b/libc/runtime/arch_prctl.c @@ -119,7 +119,7 @@ static privileged dontinline int arch_prctl_xnu(int code, int64_t addr) { case ARCH_SET_GS: asm volatile(CFLAG_ASM("syscall") : CFLAG_CONSTRAINT(failed), "=a"(ax) - : "1"(0x3000003), "D"(addr - 0x8a0 /* wat */) + : "1"(0x3000003), "D"(addr - 0x30) : "rcx", "r11", "memory", "cc"); if (failed) errno = ax, ax = -1; return ax; diff --git a/libc/runtime/clone.c b/libc/runtime/clone.c new file mode 100644 index 000000000..3b81514a7 --- /dev/null +++ b/libc/runtime/clone.c @@ -0,0 +1,535 @@ +/*-*- mode:c;indent-tabs-mode:nil;c-basic-offset:2;tab-width:8;coding:utf-8 -*-│ +│vi: set net ft=c ts=2 sts=2 sw=2 fenc=utf-8 :vi│ +╞══════════════════════════════════════════════════════════════════════════════╡ +│ Copyright 2021 Justine Alexandra Roberts Tunney │ +│ │ +│ Permission to use, copy, modify, and/or distribute this software for │ +│ any purpose with or without fee is hereby granted, provided that the │ +│ above copyright notice and this permission notice appear in all copies. │ +│ │ +│ THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL │ +│ WARRANTIES WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED │ +│ WARRANTIES OF MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE │ +│ AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL │ +│ DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR │ +│ PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER │ +│ TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR │ +│ PERFORMANCE OF THIS SOFTWARE. │ +╚─────────────────────────────────────────────────────────────────────────────*/ +#include "libc/assert.h" +#include "libc/calls/calls.h" +#include "libc/calls/internal.h" +#include "libc/calls/strace.internal.h" +#include "libc/calls/struct/ucontext-netbsd.internal.h" +#include "libc/errno.h" +#include "libc/intrin/asan.internal.h" +#include "libc/intrin/kprintf.h" +#include "libc/intrin/spinlock.h" +#include "libc/intrin/threaded.internal.h" +#include "libc/intrin/tls.h" +#include "libc/intrin/winthread.internal.h" +#include "libc/nt/runtime.h" +#include "libc/nt/thread.h" +#include "libc/nt/thunk/msabi.h" +#include "libc/runtime/runtime.h" +#include "libc/sysv/consts/clone.h" +#include "libc/sysv/consts/nr.h" +#include "libc/sysv/errfuns.h" +#include "libc/thread/freebsd.internal.h" +#include "libc/thread/xnu.internal.h" + +STATIC_YOINK("gettid"); // for kprintf() + +#define __NR_thr_new 455 +#define __NR_clone_linux 56 +#define __NR__lwp_create 309 +#define __NR_getcontext_netbsd 307 +#define __NR__lwp_setprivate 317 +#define __NR_bsdthread_create 0x02000168 +#define __NR_thread_fast_set_cthread_self 0x03000003 +#define PTHREAD_START_CUSTOM_XNU 0x01000000 +#define LWP_DETACHED 0x00000040 +#define LWP_SUSPENDED 0x00000080 + +uint32_t WinThreadThunk(void *warg); +asm(".section\t.text.windows,\"ax\",@progbits\n\t" + ".local\tWinThreadThunk\n" + "WinThreadThunk:\n\t" + "xor\t%ebp,%ebp\n\t" + "mov\t%rcx,%rdi\n\t" + "mov\t%rcx,%rsp\n\t" + "jmp\tWinThreadMain\n\t" + ".size\tWinThreadThunk,.-WinThreadThunk\n\t" + ".previous"); +__attribute__((__used__, __no_reorder__)) + +static textwindows wontreturn void +WinThreadMain(struct WinThread *wt) { + int rc; + if (wt->flags & CLONE_CHILD_SETTID) { + *wt->ctid = wt->tid; + } + // TlsSetValue(__winthread, wt); + rc = wt->func(wt->arg); + if (wt->flags & CLONE_CHILD_CLEARTID) { + *wt->ctid = 0; + } + _Exit1(rc); +} + +static textwindows int CloneWindows(int (*func)(void *), char *stk, + size_t stksz, int flags, void *arg, + int *ptid, void *tls, size_t tlssz, + int *ctid) { + int64_t h; + struct WinThread *wt; + wt = (struct WinThread *)(((intptr_t)(stk + stksz) - + sizeof(struct WinThread)) & + -alignof(struct WinThread)); + wt->flags = flags; + wt->ctid = ctid; + wt->func = func; + wt->arg = arg; + if ((h = CreateThread(0, 0, WinThreadThunk, wt, 0, &wt->tid))) { + CloseHandle(h); + if (flags & CLONE_PARENT_SETTID) { + *ptid = wt->tid; + } + return wt->tid; + } else { + __releasefd(wt->tid); + return -1; + } +} + +void XnuThreadThunk(void *pthread, int machport, void *(*func)(void *), + void *arg, intptr_t *stack, unsigned flags); +asm(".local\tXnuThreadThunk\n" + "XnuThreadThunk:\n\t" + "xor\t%ebp,%ebp\n\t" + "mov\t%r8,%rsp\n\t" + "jmp\tXnuThreadMain\n\t" + ".size\tXnuThreadThunk,.-XnuThreadThunk"); +__attribute__((__used__, __no_reorder__)) + +static wontreturn void +XnuThreadMain(void *pthread, int tid, int (*func)(void *arg), void *arg, + intptr_t *sp, unsigned flags) { + int rc; + sp[1] = tid; + _spunlock(sp); + if (sp[4] & CLONE_SETTLS) { + // XNU uses the same 0x30 offset as the WIN32 TIB x64. They told the + // Go team at Google that they Apply stands by our ability to use it + // https://github.com/golang/go/issues/23617#issuecomment-376662373 + asm volatile("syscall" + : "=a"(rc) + : "0"(__NR_thread_fast_set_cthread_self), "D"(sp[3] - 0x30) + : "rcx", "r11", "memory", "cc"); + } + if (sp[4] & CLONE_CHILD_SETTID) { + *(int *)sp[2] = tid; + } + rc = func(arg); + if (sp[4] & CLONE_CHILD_CLEARTID) { + *(int *)sp[2] = 0; + } + _Exit1(rc); +} + +static int CloneXnu(int (*fn)(void *), char *stk, size_t stksz, int flags, + void *arg, int *ptid, void *tls, size_t tlssz, int *ctid) { + int rc; + bool failed; + intptr_t *sp; + static bool once; + static int broken; + if (!once) { + if (bsdthread_register(XnuThreadThunk, 0, 0, 0, 0, 0, 0) == -1) { + broken = errno; + } + once = true; + } + if (broken) { + errno = broken; + return -1; + } + sp = (intptr_t *)(stk + stksz); + *--sp = 0; // 5 padding + *--sp = flags; // 4 clone() flags + *--sp = (intptr_t)tls; // 3 thread local storage + *--sp = (intptr_t)ctid; // 2 child tid api + *--sp = 0; // 1 receives tid + *--sp = 0; // 0 lock + _seizelock(sp); // TODO: How can we get the tid without locking? + if ((rc = bsdthread_create(fn, arg, sp, 0, PTHREAD_START_CUSTOM_XNU)) != -1) { + _spinlock(sp); + if (flags & CLONE_PARENT_SETTID) { + *ptid = sp[1]; + } + rc = sp[1]; + } + return rc; +} + +void FreebsdThreadThunk(void *sp) wontreturn; +asm(".local\tFreebsdThreadThunk\n" + "FreebsdThreadThunk:\n\t" + "xor\t%ebp,%ebp\n\t" + "mov\t%rdi,%rsp\n\t" + "jmp\tFreebsdThreadMain\n\t" + ".size\tFreebsdThreadThunk,.-FreebsdThreadThunk"); +__attribute__((__used__, __no_reorder__)) + +static wontreturn void +FreebsdThreadMain(intptr_t *sp) { + int rc; + if (sp[3] & CLONE_CHILD_SETTID) { + *(int *)sp[2] = sp[4]; + } + rc = ((int (*)(intptr_t))sp[0])(sp[1]); + if (sp[3] & CLONE_CHILD_CLEARTID) { + *(int *)sp[2] = 0; + } + _Exit1(rc); +} + +static int CloneFreebsd(int (*func)(void *), char *stk, size_t stksz, int flags, + void *arg, int *ptid, void *tls, size_t tlssz, + int *ctid) { + int ax; + bool failed; + int64_t tid; + intptr_t *sp; + sp = (intptr_t *)(stk + stksz); + *--sp = 0; // 5 [padding] + *--sp = 0; // 4 [child_tid] + *--sp = flags; // 3 + *--sp = (intptr_t)ctid; // 2 + *--sp = (intptr_t)arg; // 1 + *--sp = (intptr_t)func; // 0 + struct thr_param params = { + .start_func = FreebsdThreadThunk, + .arg = sp, + .stack_base = stk, + .stack_size = stksz, + .tls_base = flags & CLONE_SETTLS ? tls : 0, + .tls_size = flags & CLONE_SETTLS ? tlssz : 0, + .child_tid = sp + 4, + .parent_tid = &tid, + }; + asm volatile(CFLAG_ASM("syscall") + : CFLAG_CONSTRAINT(failed), "=a"(ax) + : "1"(__NR_thr_new), "D"(¶ms), "S"(sizeof(params)) + : "rcx", "rdx", "r8", "r9", "r10", "r11", "memory"); + if (!failed) { + if (flags & CLONE_PARENT_SETTID) { + *ptid = tid; + } + return tid; + } else { + errno = ax; + return -1; + } +} + +struct __tfork { + void *tf_tcb; + int32_t *tf_tid; + void *tf_stack; +}; + +int __tfork(struct __tfork *params, size_t psize, intptr_t *stack); +asm(".section\t.privileged,\"ax\",@progbits\n\t" + ".local\t__tfork\n" + "__tfork:\n\t" + "push\t$8\n\t" + "pop\t%rax\n\t" + "mov\t%rdx,%r8\n\t" + "syscall\n\t" + "jc\t1f\n\t" + "test\t%eax,%eax\n\t" + "jz\t2f\n\t" + "ret\n1:\t" + "neg\t%eax\n\t" + "ret\n2:\t" + "xor\t%ebp,%ebp\n\t" + "mov\t%r8,%rsp\n\t" + "mov\t%r8,%rdi\n\t" + "jmp\tOpenbsdThreadMain\n\t" + ".size\t__tfork,.-__tfork\n\t" + ".previous"); +__attribute__((__used__, __no_reorder__)) + +static privileged wontreturn void +OpenbsdThreadMain(intptr_t *sp) { + int rc; + rc = ((int (*)(intptr_t))sp[0])(sp[1]); + if (sp[3] & CLONE_CHILD_CLEARTID) { + *(int *)sp[2] = 0; + } + _Exit1(rc); +} + +static int CloneOpenbsd(int (*func)(void *), char *stk, size_t stksz, int flags, + void *arg, int *ptid, void *tls, size_t tlssz, + int *ctid) { + int tid; + intptr_t *sp; + struct __tfork params; + sp = (intptr_t *)(stk + stksz); + *--sp = flags; // 3 + *--sp = (intptr_t)ctid; // 2 + *--sp = (intptr_t)arg; // 1 + *--sp = (intptr_t)func; // 0 + params.tf_stack = sp; + params.tf_tcb = flags & CLONE_SETTLS ? tls : 0; + params.tf_tid = flags & CLONE_CHILD_SETTID ? ctid : 0; + if ((tid = __tfork(¶ms, sizeof(params), sp)) > 0) { + if (flags & CLONE_PARENT_SETTID) { + *ptid = tid; + } + } else { + errno = -tid; + tid = -1; + } + return tid; +} + +static wontreturn void NetbsdThreadMain(void *arg, int (*func)(void *arg), + int *tid, int *ctid, int flags) { + int rc; + if (flags & CLONE_CHILD_SETTID) { + *ctid = *tid; + } + rc = func(arg); + if (flags & CLONE_CHILD_CLEARTID) { + *ctid = 0; + } + _Exit1(rc); +} + +static int CloneNetbsd(int (*func)(void *), char *stk, size_t stksz, int flags, + void *arg, int *ptid, void *tls, size_t tlssz, + int *ctid) { + // NetBSD has its own clone() and it works, but it's technically a + // second-class API, intended to help Linux folks migrate to this! + // We put it on the thread's stack, to avoid locking this function + // so its stack doesn't scope. The ucontext struct needs 784 bytes + bool failed; + int ax, *tid; + intptr_t dx, sp; + static bool once; + static int broken; + struct ucontext_netbsd *ctx; + static struct ucontext_netbsd netbsd_clone_template; + if (!once) { + asm volatile(CFLAG_ASM("syscall") + : CFLAG_CONSTRAINT(failed), "=a"(ax) + : "1"(__NR_getcontext_netbsd), "D"(&netbsd_clone_template) + : "rcx", "rdx", "r11", "memory"); + if (failed) { + broken = ax; + } + once = true; + } + if (broken) { + errno = broken; + return -1; + } + sp = (intptr_t)(stk + stksz); + sp -= sizeof(int); + tid = (int *)sp; + sp -= sizeof(*ctx); + sp = sp & -alignof(*ctx); + ctx = (struct ucontext_netbsd *)sp; + memcpy(ctx, &netbsd_clone_template, sizeof(*ctx)); + ctx->uc_link = 0; + ctx->uc_mcontext.rbp = 0; + ctx->uc_mcontext.rsp = sp; + ctx->uc_mcontext.rip = (intptr_t)NetbsdThreadMain; + ctx->uc_mcontext.rdi = (intptr_t)arg; + ctx->uc_mcontext.rsi = (intptr_t)func; + ctx->uc_mcontext.rdx = (intptr_t)tid; + ctx->uc_mcontext.rcx = (intptr_t)ctid; + ctx->uc_mcontext.r8 = flags; + ctx->uc_flags |= _UC_STACK; + ctx->uc_stack.ss_sp = stk; + ctx->uc_stack.ss_size = stksz; + ctx->uc_stack.ss_flags = 0; + if (flags & CLONE_SETTLS) { + ctx->uc_flags |= _UC_TLSBASE; + ctx->uc_mcontext._mc_tlsbase = (intptr_t)tls; + } + asm volatile(CFLAG_ASM("syscall") + : CFLAG_CONSTRAINT(failed), "=a"(ax), "=d"(dx) + : "1"(__NR__lwp_create), "D"(ctx), "S"(LWP_DETACHED), "2"(tid) + : "rcx", "r11", "memory"); + if (!failed) { + if (flags & CLONE_PARENT_SETTID) { + *ptid = *tid; + } + return *tid; + } else { + errno = ax; + return -1; + } +} + +static int CloneLinux(int (*func)(void *), char *stk, size_t stksz, int flags, + void *arg, int *ptid, void *tls, size_t tlssz, + int *ctid) { + int ax; + bool failed; + intptr_t *stack; + register int *r8 asm("r8") = tls; + register int (*r9)(void *) asm("r9") = func; + register int *r10 asm("r10") = ctid; + stack = (intptr_t *)(stk + stksz); + *--stack = (long)arg; // push 1 + asm volatile("syscall" + : "=a"(ax) + : "0"(__NR_clone_linux), "D"(flags), "S"(stack), "d"(ptid), + "r"(r10), "r"(r8), "r"(r9) + : "rcx", "r11", "memory"); + if (ax > -4096u) { + errno = -ax; + return -1; + } + if (ax) return ax; + asm volatile("xor\t%%ebp,%%ebp\n\t" + "pop\t%%rdi\n\t" // pop 1 + "call\t*%0\n\t" + "xchg\t%%eax,%%edi\n\t" + "jmp\t_Exit1" + : /* no outputs */ + : "r"(r9) + : "memory"); + unreachable; +} + +/** + * Creates thread. + * + * Threads are created in a detached manner. They currently can't be + * synchronized using wait() and posix signals. Threads created by this + * function should be synchronized using shared memory operations. + * + * Any memory that's required by this system call wrapper is allocated + * to the top of your stack. This is normally about 64 bytes, although + * on NetBSD it's currently 800. + * + * This function follows the same ABI convention as the Linux userspace + * libraries, with a few small changes. The varargs has been removed to + * help prevent broken code, and the stack size and tls size parameters + * are introduced for compatibility with FreeBSD. + * + * @param func is your callback function + * @param stk points to the bottom of a caller allocated stack, which + * must be null when fork() and vfork() equivalent flags are used + * and furthermore this must be mmap()'d using MAP_STACK in order + * to work on OpenBSD + * @param stksz is the size of that stack in bytes which must be zero + * if the fork() or vfork() equivalent flags are used it's highly + * recommended that this value be GetStackSize(), or else kprintf + * and other runtime services providing memory safety can't do as + * good and quick of a job; this value must be 4096-aligned, plus + * it must be at minimum 4096 bytes in size + * @param flags usually has one of + * - `SIGCHLD` will delegate to fork() + * - `CLONE_VFORK|CLONE_VM|SIGCHLD` means vfork() + * - `CLONE_VM|CLONE_FS|CLONE_FILES|CLONE_SIGHAND` for threads + * as part high bytes, and the low order byte may optionally contain + * a signal e.g. SIGCHLD, to enable parent notification on terminate + * although the signal isn't supported on non-Linux and non-NetBSD + * at the moment; 'flags' may optionally bitwise or the following: + * - `CLONE_PARENT_SETTID` is needed for `ctid` should be set + * - `CLONE_CHILD_SETTID` is needed for `ptid` should be set + * - `CLONE_SETTLS` is needed to set `%fs` segment to `tls` + * @param arg will be passed to your callback + * @param ptid lets the parent receive the child thread id; + * this parameter is ignored if `CLONE_PARENT_SETTID` is not set + * @param tls may be used to set the thread local storage segment; + * this parameter is ignored if `CLONE_SETTLS` is not set + * @param tlssz is the size of tls in bytes + * @param ctid lets the child receive its thread id; + * this parameter is ignored if `CLONE_CHILD_SETTID` is not set + * @return tid on success and 0 to the child, otherwise -1 w/ errno + * @threadsafe + */ +int clone(int (*func)(void *), void *stk, size_t stksz, int flags, void *arg, + int *ptid, void *tls, size_t tlssz, int *ctid) { + int rc; + + // let kprintf() switch from pids to tids + __threaded = true; + + // verify memory is kosher + if (IsAsan() && + ((stksz > PAGESIZE && + !__asan_is_valid((char *)stk + PAGESIZE, stksz - PAGESIZE)) || + ((flags & CLONE_SETTLS) && !__asan_is_valid(tls, tlssz)) || + ((flags & CLONE_SETTLS) && !__asan_is_valid(tls, sizeof(long))) || + ((flags & CLONE_PARENT_SETTID) && + !__asan_is_valid(ptid, sizeof(*ptid))) || + ((flags & CLONE_CHILD_SETTID) && + !__asan_is_valid(ctid, sizeof(*ctid))))) { + rc = efault(); + } + + // delegate to bona fide clone() + else if (IsLinux()) { + rc = CloneLinux(func, stk, stksz, flags, arg, ptid, tls, tlssz, ctid); + } + + // polyfill fork() and vfork() use cases on platforms without clone() + else if ((SupportsWindows() || SupportsBsd()) && + flags == (CLONE_VFORK | CLONE_VM | SIGCHLD)) { + if (IsTiny()) { + rc = einval(); + } else if (!arg && !stksz) { + return vfork(); // don't log clone() + } else { + rc = einval(); + } + } else if ((SupportsWindows() || SupportsBsd()) && flags == SIGCHLD) { + if (IsTiny()) { + rc = eopnotsupp(); + } else if (!arg && !stksz) { + return fork(); // don't log clone() + } else { + rc = einval(); + } + } + + // we now assume we're creating a thread + // these platforms can't do signals the way linux does + else if (!IsTiny() && + ((stksz < PAGESIZE || (stksz & (PAGESIZE - 1))) || + (flags & + ~(CLONE_SETTLS | CLONE_PARENT_SETTID | CLONE_CHILD_SETTID)) != + (CLONE_VM | CLONE_FS | CLONE_FILES | CLONE_SIGHAND))) { + rc = einval(); + } else if (IsXnu()) { + rc = CloneXnu(func, stk, stksz, flags, arg, ptid, tls, tlssz, ctid); + } else if (IsFreebsd()) { + rc = CloneFreebsd(func, stk, stksz, flags, arg, ptid, tls, tlssz, ctid); + } else if (IsNetbsd()) { + rc = CloneNetbsd(func, stk, stksz, flags, arg, ptid, tls, tlssz, ctid); + } else if (IsOpenbsd()) { + rc = CloneOpenbsd(func, stk, stksz, flags, arg, ptid, tls, tlssz, ctid); + } + + // These platforms can't do segment registers like linux does + else if (flags & CLONE_SETTLS) { + rc = einval(); + } else if (IsWindows()) { + rc = CloneWindows(func, stk, stksz, flags, arg, ptid, tls, tlssz, ctid); + } else { + rc = enosys(); + } + + STRACE("clone(%p, %p, %'zu, %#x, %p, %p, %p, %'zu, %p) → %d% m", func, stk, + stksz, flags, arg, ptid, tls, tlssz, ctid, rc); + return rc; +} diff --git a/libc/runtime/fork-nt.c b/libc/runtime/fork-nt.c index 217af382b..e737f6f3e 100644 --- a/libc/runtime/fork-nt.c +++ b/libc/runtime/fork-nt.c @@ -228,8 +228,8 @@ textwindows void WinMainForked(void) { fds->p[1].handle = fds->__init_p[1].handle = GetStdHandle(kNtStdOutputHandle); fds->p[2].handle = fds->__init_p[2].handle = GetStdHandle(kNtStdErrorHandle); - // untrack the forked children of the parent since we marked the - // CreateProcess() process handle below as non-inheritable + // untrack children of parent since we specify with both + // CreateProcess() and CreateThread() as non-inheritable for (i = 0; i < fds->n; ++i) { if (fds->p[i].kind == kFdProcess) { fds->p[i].kind = 0; diff --git a/libc/runtime/mmap.c b/libc/runtime/mmap.c index e39f2f6fe..47bc1527d 100644 --- a/libc/runtime/mmap.c +++ b/libc/runtime/mmap.c @@ -25,6 +25,7 @@ #include "libc/dce.h" #include "libc/errno.h" #include "libc/intrin/asan.internal.h" +#include "libc/intrin/asancodes.h" #include "libc/intrin/describeflags.internal.h" #include "libc/intrin/kprintf.h" #include "libc/intrin/spinlock.h" @@ -381,6 +382,10 @@ static noasan inline void *Mmap(void *addr, size_t size, int prot, int flags, if (needguard) { if (IsWindows()) _spunlock(&_mmi.lock); mprotect(p, PAGESIZE, PROT_NONE); + if (IsAsan()) { + __repstosb((void *)(((intptr_t)p >> 3) + 0x7fff8000), + kAsanStackOverflow, PAGESIZE / 8); + } if (IsWindows()) _spinlock(&_mmi.lock); } } @@ -408,7 +413,29 @@ static noasan inline void *Mmap(void *addr, size_t size, int prot, int flags, * will be rounded up to FRAMESIZE automatically if MAP_ANONYMOUS * is specified * @param prot can have PROT_READ/PROT_WRITE/PROT_EXEC/PROT_NONE/etc. - * @param flags can have MAP_ANONYMOUS, MAP_SHARED, MAP_PRIVATE, etc. + * @param flags should have one of the following masked by `MAP_TYPE` + * - `MAP_FILE` in which case `fd != -1` should be the case + * - `MAP_PRIVATE` for copy-on-write behavior of writeable pages + * - `MAP_SHARED` to create shared memory between processes + * - `MAP_STACK` to create a grows-down alloc, where a guard page + * is automatically protected at the bottom: FreeBSD's behavior + * is polyfilled across platforms; uses MAP_GROWSDOWN on Linux + * too for extra oomph (do not use MAP_GROWSDOWN!) and this is + * completely mandatory on OpenBSD but helps perf elsewhere if + * you need to create 10,000 threads. This flag is the reason + * why `STACK_FRAME_UNLIMITED` toil is important, because this + * only allocates a 4096-byte guard page, thus we need the GCC + * compile-time checks to ensure some char[8192] vars will not + * create an undetectable overflow into another thread's stack + * Your `flags` may optionally bitwise or any of the following: + * - `MAP_FIXED` in which case `addr` becomes more than a hint + * - `MAP_FIXED_NOREPLACE` to protect existing maps (Linux-only) + * - `MAP_ANONYMOUS` in which case `fd == -1` should be the case + * - `MAP_CONCEAL` is FreeBSD/NetBSD/OpenBSD-only + * - `MAP_NORESERVE` is Linux/XNU/NetBSD-only + * - `MAP_LOCKED` is Linux-only + * - `MAP_POPULATE` is Linux-only + * - `MAP_NONBLOCK` is Linux-only * @param fd is an open()'d file descriptor, whose contents shall be * made available w/ automatic reading at the chosen address and * must be -1 if MAP_ANONYMOUS is specified diff --git a/libc/sysv/calls/__tfork.s b/libc/sysv/calls/__tfork.s deleted file mode 100644 index 6ae6b1f1e..000000000 --- a/libc/sysv/calls/__tfork.s +++ /dev/null @@ -1,2 +0,0 @@ -.include "o/libc/sysv/macros.internal.inc" -.scall __tfork,0xfff008ffffffffff,globl diff --git a/libc/sysv/calls/getcontext.s b/libc/sysv/calls/getcontext.s deleted file mode 100644 index d21d6096b..000000000 --- a/libc/sysv/calls/getcontext.s +++ /dev/null @@ -1,2 +0,0 @@ -.include "o/libc/sysv/macros.internal.inc" -.scall getcontext,0x133fff1a5fffffff,globl diff --git a/libc/sysv/calls/sys_getcontext.s b/libc/sysv/calls/sys_getcontext.s new file mode 100644 index 000000000..9569e7596 --- /dev/null +++ b/libc/sysv/calls/sys_getcontext.s @@ -0,0 +1,2 @@ +.include "o/libc/sysv/macros.internal.inc" +.scall sys_getcontext,0x133fff1a5fffffff,globl,hidden diff --git a/libc/sysv/calls/sys_tgkill.s b/libc/sysv/calls/sys_tgkill.s new file mode 100644 index 000000000..18b2b2a38 --- /dev/null +++ b/libc/sysv/calls/sys_tgkill.s @@ -0,0 +1,2 @@ +.include "o/libc/sysv/macros.internal.inc" +.scall sys_tgkill,0xfffffffffffff0ea,globl,hidden diff --git a/libc/sysv/calls/sys_tkill.s b/libc/sysv/calls/sys_tkill.s new file mode 100644 index 000000000..47e2c444f --- /dev/null +++ b/libc/sysv/calls/sys_tkill.s @@ -0,0 +1,2 @@ +.include "o/libc/sysv/macros.internal.inc" +.scall sys_tkill,0x13e0771b121690c8,globl,hidden diff --git a/libc/sysv/calls/tgkill.s b/libc/sysv/calls/tgkill.s deleted file mode 100644 index d5c1cd7d4..000000000 --- a/libc/sysv/calls/tgkill.s +++ /dev/null @@ -1,2 +0,0 @@ -.include "o/libc/sysv/macros.internal.inc" -.scall tgkill,0xfffffffffffff0ea,globl diff --git a/libc/sysv/calls/tkill.s b/libc/sysv/calls/tkill.s deleted file mode 100644 index d11f0d151..000000000 --- a/libc/sysv/calls/tkill.s +++ /dev/null @@ -1,2 +0,0 @@ -.include "o/libc/sysv/macros.internal.inc" -.scall tkill,0xfffffffffffff0c8,globl diff --git a/libc/sysv/consts.sh b/libc/sysv/consts.sh index cc73c086b..2ad3f7c43 100755 --- a/libc/sysv/consts.sh +++ b/libc/sysv/consts.sh @@ -218,7 +218,7 @@ syscon compat O_LARGEFILE 0 0 0 0 0 0 # # the revolutionary praxis of malloc() # # group name GNU/Systemd XNU's Not UNIX! FreeBSD OpenBSD NetBSD The New Technology Commentary -syscon compat MAP_FILE 0 0 0 0 0 0 # consensus +syscon mmap MAP_FILE 0 0 0 0 0 0 # consensus syscon mmap MAP_SHARED 1 1 1 1 1 1 # forced consensus & faked nt syscon mmap MAP_PRIVATE 2 2 2 2 2 2 # forced consensus & faked nt syscon mmap MAP_STACK 6 6 6 6 6 6 # our definition @@ -2318,9 +2318,7 @@ syscon nr __NR_access_extended 0xfff 0x200011c 0xfff 0xfff 0xfff 0xff syscon nr __NR_audit_session_join 0xfff 0x20001ad 0xfff 0xfff 0xfff 0xfff syscon nr __NR_audit_session_port 0xfff 0x20001b0 0xfff 0xfff 0xfff 0xfff syscon nr __NR_audit_session_self 0xfff 0x20001ac 0xfff 0xfff 0xfff 0xfff -syscon nr __NR_bsdthread_create 0xfff 0x2000168 0xfff 0xfff 0xfff 0xfff syscon nr __NR_bsdthread_ctl 0xfff 0x20001de 0xfff 0xfff 0xfff 0xfff -syscon nr __NR_bsdthread_register 0xfff 0x200016e 0xfff 0xfff 0xfff 0xfff syscon nr __NR_bsdthread_terminate 0xfff 0x2000169 0xfff 0xfff 0xfff 0xfff syscon nr __NR_change_fdguard_np 0xfff 0x20001bc 0xfff 0xfff 0xfff 0xfff syscon nr __NR_chmod_extended 0xfff 0x200011a 0xfff 0xfff 0xfff 0xfff diff --git a/libc/sysv/consts/__NR_bsdthread_create.S b/libc/sysv/consts/__NR_bsdthread_create.S deleted file mode 100644 index 149b7ee66..000000000 --- a/libc/sysv/consts/__NR_bsdthread_create.S +++ /dev/null @@ -1,2 +0,0 @@ -#include "libc/sysv/consts/syscon.internal.h" -.syscon nr,__NR_bsdthread_create,0xfff,0x2000168,0xfff,0xfff,0xfff,0xfff diff --git a/libc/sysv/consts/__NR_bsdthread_register.S b/libc/sysv/consts/__NR_bsdthread_register.S deleted file mode 100644 index 8a81f93c4..000000000 --- a/libc/sysv/consts/__NR_bsdthread_register.S +++ /dev/null @@ -1,2 +0,0 @@ -#include "libc/sysv/consts/syscon.internal.h" -.syscon nr,__NR_bsdthread_register,0xfff,0x200016e,0xfff,0xfff,0xfff,0xfff diff --git a/libc/sysv/consts/nr.h b/libc/sysv/consts/nr.h index 00ed55a70..c926b9dcc 100644 --- a/libc/sysv/consts/nr.h +++ b/libc/sysv/consts/nr.h @@ -404,9 +404,7 @@ extern const long __NR_access_extended; extern const long __NR_audit_session_join; extern const long __NR_audit_session_port; extern const long __NR_audit_session_self; -extern const long __NR_bsdthread_create; extern const long __NR_bsdthread_ctl; -extern const long __NR_bsdthread_register; extern const long __NR_bsdthread_terminate; extern const long __NR_change_fdguard_np; extern const long __NR_chmod_extended; @@ -1101,7 +1099,6 @@ COSMOPOLITAN_C_END_ #define __NR_pidfd_send_signal SYMBOLIC(__NR_pidfd_send_signal) #define __NR_io_uring_setup SYMBOLIC(__NR_io_uring_setup) #define __NR_io_uring_enter SYMBOLIC(__NR_io_uring_enter) -#define __NR_io_uring_register SYMBOLIC(__NR_io_uring_register) #define __NR_pledge SYMBOLIC(__NR_pledge) #define __NR_msyscall SYMBOLIC(__NR_msyscall) #define __NR_ktrace SYMBOLIC(__NR_ktrace) @@ -1173,7 +1170,6 @@ COSMOPOLITAN_C_END_ #define __NR_audit_session_join SYMBOLIC(__NR_audit_session_join) #define __NR_audit_session_port SYMBOLIC(__NR_audit_session_port) #define __NR_audit_session_self SYMBOLIC(__NR_audit_session_self) -#define __NR_bsdthread_create SYMBOLIC(__NR_bsdthread_create) #define __NR_bsdthread_ctl SYMBOLIC(__NR_bsdthread_ctl) #define __NR_bsdthread_register SYMBOLIC(__NR_bsdthread_register) #define __NR_bsdthread_terminate SYMBOLIC(__NR_bsdthread_terminate) diff --git a/libc/sysv/syscalls.sh b/libc/sysv/syscalls.sh index 8c4ff493d..e74f31739 100755 --- a/libc/sysv/syscalls.sh +++ b/libc/sysv/syscalls.sh @@ -99,7 +99,7 @@ scall __sys_wait4 0x1c100b007200703d globl hidden scall sys_kill 0x02507a025202503e globl hidden # kill(pid, sig, 1) b/c xnu scall sys_killpg 0xffffff092fffffff globl hidden scall sys_clone 0x11fffffffffff038 globl hidden -scall tkill 0xfffffffffffff0c8 globl +scall sys_tkill 0x13e0771b121690c8 globl hidden # thr_kill() on freebsd; _lwp_kill() on netbsd; thrkill() on openbsd where arg3 should be 0; bsdthread_terminate() on XNU which only has 1 arg scall futex 0xfff053fffffff0ca globl scall set_robust_list 0xfffffffffffff111 globl scall get_robust_list 0xfffffffffffff112 globl @@ -265,7 +265,7 @@ scall clock_settime 0x1ac0580e9ffff0e3 globl scall sys_clock_gettime 0x1ab0570e8ffff0e4 globl hidden # Linux 2.6+ (c. 2003); XNU uses magic address scall clock_getres 0x1ad0590eaffff0e5 globl scall clock_nanosleep 0xffffff0f4ffff0e6 globl -scall tgkill 0xfffffffffffff0ea globl +scall sys_tgkill 0xfffffffffffff0ea globl hidden scall mbind 0xfffffffffffff0ed globl scall set_mempolicy 0xfffffffffffff0ee globl scall get_mempolicy 0xfffffffffffff0ef globl @@ -658,7 +658,7 @@ scall fhlink 0xffffff235fffffff globl scall fhlinkat 0xffffff236fffffff globl scall fhreadlink 0xffffff237fffffff globl scall getaudit 0xffffff1c1fffffff globl -scall getcontext 0x133fff1a5fffffff globl +scall sys_getcontext 0x133fff1a5fffffff globl hidden #scall getdomainname 0xffff00a2ffffffff globl scall getfhat 0xffffff234fffffff globl scall gethostid 0xffffff08efffffff globl @@ -756,7 +756,6 @@ scall wait 0xffffff054fffffff globl scall wait6 0x1e1fff214fffffff globl scall yield 0xffffff141fffffff globl #──────────────────────────OPENBSD─────────────────────────── -scall __tfork 0xfff008ffffffffff globl scall __thrsleep 0xfff05effffffffff globl scall __thrwakeup 0xfff12dffffffffff globl scall __threxit 0xfff12effffffffff globl diff --git a/libc/thread/clone.c b/libc/thread/clone.c deleted file mode 100644 index bdb666c32..000000000 --- a/libc/thread/clone.c +++ /dev/null @@ -1,400 +0,0 @@ -/*-*- mode:c;indent-tabs-mode:nil;c-basic-offset:2;tab-width:8;coding:utf-8 -*-│ -│vi: set net ft=c ts=2 sts=2 sw=2 fenc=utf-8 :vi│ -╞══════════════════════════════════════════════════════════════════════════════╡ -│ Copyright 2021 Justine Alexandra Roberts Tunney │ -│ │ -│ Permission to use, copy, modify, and/or distribute this software for │ -│ any purpose with or without fee is hereby granted, provided that the │ -│ above copyright notice and this permission notice appear in all copies. │ -│ │ -│ THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL │ -│ WARRANTIES WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED │ -│ WARRANTIES OF MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE │ -│ AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL │ -│ DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR │ -│ PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER │ -│ TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR │ -│ PERFORMANCE OF THIS SOFTWARE. │ -╚─────────────────────────────────────────────────────────────────────────────*/ -#include "libc/bits/asmflag.h" -#include "libc/bits/weaken.h" -#include "libc/calls/internal.h" -#include "libc/calls/sig.internal.h" -#include "libc/calls/strace.internal.h" -#include "libc/calls/struct/ucontext-netbsd.internal.h" -#include "libc/dce.h" -#include "libc/errno.h" -#include "libc/intrin/asan.internal.h" -#include "libc/intrin/spinlock.h" -#include "libc/intrin/tls.h" -#include "libc/intrin/winthread.internal.h" -#include "libc/mem/mem.h" -#include "libc/nexgen32e/nt2sysv.h" -#include "libc/nexgen32e/stackframe.h" -#include "libc/nt/runtime.h" -#include "libc/nt/thread.h" -#include "libc/runtime/runtime.h" -#include "libc/sysv/consts/clone.h" -#include "libc/sysv/consts/o.h" -#include "libc/sysv/consts/sicode.h" -#include "libc/sysv/consts/sig.h" -#include "libc/sysv/errfuns.h" -#include "libc/thread/freebsd.internal.h" -#include "libc/thread/openbsd.internal.h" - -// TODO(jart): work in progress - -STATIC_YOINK("gettid"); // for kprintf() - -#define __NR_thr_new 455 -#define __NR___tfork 8 -#define __NR_clone_linux 56 -#define __NR__lwp_create 309 -#define __NR_getcontext_netbsd 307 -#define __NR__lwp_setprivate 317 - -extern bool __threaded; - -static struct Cloner { - _Alignas(64) char lock; - _Alignas(64) int flags; - int64_t tid; - int (*func)(void *); - void *arg; - void *stack; - int *ctid; - int *ptid; -} __cloner; - -static textwindows uint32_t WinThreadMain(void *notused) { - intptr_t rdi, rdx; - int (*func)(void *); - void *arg, *stack; - struct WinThread *wt; - int exitcode, tid, flags, *ctid; - tid = __cloner.tid; - arg = __cloner.arg; - func = __cloner.func; - ctid = __cloner.ctid; - flags = __cloner.flags; - stack = __cloner.stack; - _spunlock(&__cloner.lock); - wt = calloc(1, sizeof(struct WinThread)); - wt->pid = tid; - TlsSetValue(__winthread, wt); - if (flags & CLONE_CHILD_SETTID) *ctid = tid; - asm volatile("push\t%%rbp\n\t" - "mov\t%%rsp,%%r15\n\t" - "xor\t%%ebp,%%ebp\n\t" - "xchg\t%%rax,%%rsp\n\t" - "call\t*%2\n\t" - "mov\t%%rbx,%%rbp\n\t" - "mov\t%%r15,%%rsp\n\t" - "pop\t%%rbp" - : "=a"(exitcode), "=D"(rdi), "=d"(rdx) - : "0"(stack), "1"(arg), "2"(func) - : "rbx", "rcx", "rsi", "r8", "r9", "r10", "r11", "r15", - "memory"); - if (flags & CLONE_CHILD_CLEARTID) *ctid = 0; - __releasefd(tid); - free(wt); - return exitcode; -} - -static textwindows int CloneWindows(int (*func)(void *), void *stk, - size_t stksz, int flags, void *arg, - int *ptid, void *tls, size_t tlssz, - int *ctid) { - int tid; - int64_t hand; - uint32_t wintid; - if ((tid = __reservefd(-1)) == -1) return -1; - _spinlock(&__cloner.lock); - __cloner.tid = tid; - __cloner.arg = arg; - __cloner.func = func; - __cloner.ctid = ctid; - __cloner.flags = flags; - __cloner.stack = (char *)stk + stksz; - if (!(hand = CreateThread(0, 0, NT2SYSV(WinThreadMain), 0, 0, &wintid))) { - _spunlock(&__cloner.lock); - return -1; - } - if (flags & CLONE_CHILD_SETTID) *ctid = tid; - if (flags & CLONE_PARENT_SETTID) *ptid = tid; - // XXX: this should be tracked in a separate data structure - g_fds.p[tid].kind = kFdProcess; - g_fds.p[tid].handle = hand; - g_fds.p[tid].flags = O_CLOEXEC; - g_fds.p[tid].zombie = false; - return tid; -} - -static dontinline wontreturn void BsdThreadMain(void *unused) { - void *arg; - int (*func)(void *); - int tid, flags, exitcode, *ctid; - asm("xor\t%ebp,%ebp"); - tid = __cloner.tid; - arg = __cloner.arg; - func = __cloner.func; - ctid = __cloner.ctid; - flags = __cloner.flags; - _spunlock(&__cloner.lock); - if (flags & CLONE_CHILD_SETTID) *ctid = tid; - exitcode = func(arg); - if (flags & CLONE_CHILD_CLEARTID) *ctid = 0; - _Exit1(exitcode); -} - -static privileged noasan int CloneFreebsd(int (*func)(void *), void *stk, - size_t stksz, int flags, void *arg, - int *ptid, void *tls, size_t tlssz, - int *ctid) { - int ax; - bool failed; - int64_t tid; - struct thr_param params = {0}; - _spinlock(&__cloner.lock); - __cloner.arg = arg; - __cloner.func = func; - __cloner.ctid = ctid; - __cloner.flags = flags; - params.start_func = BsdThreadMain; - params.stack_base = stk; - params.stack_size = stksz; - params.tls_base = flags & CLONE_SETTLS ? tls : 0; - params.tls_size = flags & CLONE_SETTLS ? tlssz : 0; - params.child_tid = &__cloner.tid; - params.parent_tid = &tid; - asm volatile(CFLAG_ASM("syscall") - : CFLAG_CONSTRAINT(failed), "=a"(ax) - : "1"(__NR_thr_new), "D"(¶ms), "S"(sizeof(params)) - : "rcx", "r11", "memory", "cc"); - if (!failed) { - if (flags & CLONE_PARENT_SETTID) *ptid = tid; - return tid; - } else { - errno = ax; - return -1; - } -} - -static privileged noasan int CloneOpenbsd(int (*func)(void *), char *stk, - size_t stksz, int flags, void *arg, - int *ptid, void *tls, size_t tlssz, - int *ctid) { - int ax; - bool failed; - struct __tfork params; - _spinlock(&__cloner.lock); - __cloner.arg = arg; - __cloner.func = func; - __cloner.ctid = ctid; - __cloner.flags = flags; - __cloner.tid = 0; - asm volatile("" ::: "memory"); - params.tf_tid = (int *)&__cloner.tid; - params.tf_tcb = flags & CLONE_SETTLS ? tls : 0; - // we need openbsd:stackbound because openbsd kernel enforces rsp must - // be on interval [stack, stack+size) thus the top address is an error - // furthermore this needs to be allocated using MAP_STACK OR GROWSDOWN - params.tf_stack = (void *)((intptr_t)((char *)stk + stksz - 1) & -16); - asm volatile(CFLAG_ASM("syscall") - : CFLAG_CONSTRAINT(failed), "=a"(ax) - : "1"(__NR___tfork), "D"(¶ms), "S"(sizeof(params)) - : "r11", "memory", "cc"); - if (failed) { - errno = ax; - return -1; - } - if (ax) { - if (flags & CLONE_PARENT_SETTID) *ptid = ax; - return ax; - } - BsdThreadMain(0); - unreachable; -} - -static privileged noasan int CloneNetbsd(int (*func)(void *), void *stk, - size_t stksz, int flags, void *arg, - int *ptid, void *tls, size_t tlssz, - int *ctid) { - int ax, tid; - bool failed; - intptr_t *stack; - struct ucontext_netbsd ctx; - asm volatile(CFLAG_ASM("syscall") - : CFLAG_CONSTRAINT(failed), "=a"(ax) - : "1"(__NR_getcontext_netbsd), "D"(&ctx) - : "rcx", "r11", "memory", "cc"); - if (failed) { - errno = ax; - return -1; - } - stack = (intptr_t *)((intptr_t)((char *)stk + stksz) & -16); - *--stack = (intptr_t)_Exit1; - ctx.uc_link = 0; - ctx.uc_mcontext.rip = (intptr_t)func; - ctx.uc_mcontext.rdi = (intptr_t)arg; - ctx.uc_mcontext.rsp = (intptr_t)stack; - ctx.uc_mcontext.rbp = 0; - ctx.uc_flags |= _UC_STACK; - ctx.uc_stack.ss_sp = stk; - ctx.uc_stack.ss_size = stksz; - ctx.uc_stack.ss_flags = 0; - if (flags & CLONE_SETTLS) { - ctx.uc_flags |= _UC_TLSBASE; - ctx.uc_mcontext._mc_tlsbase = (intptr_t)tls; - } - asm volatile("" ::: "memory"); - asm volatile(CFLAG_ASM("syscall") - : CFLAG_CONSTRAINT(failed), "=a"(ax) - : "1"(__NR__lwp_create), "D"(&ctx), "S"(flags), "d"(&tid) - : "rcx", "r11", "memory", "cc"); - if (failed) { - errno = ax; - return -1; - } - if (flags & CLONE_PARENT_SETTID) *ptid = ax; - if (flags & CLONE_CHILD_SETTID) *ctid = ax; - return tid; -} - -static privileged int CloneLinux(int (*func)(void *), void *stk, size_t stksz, - int flags, void *arg, int *ptid, void *tls, - size_t tlssz, int *ctid) { - int ax; - bool failed; - intptr_t *stack; - register int *r8 asm("r8") = tls; - register int (*r9)(void *) asm("r9") = func; - register int *r10 asm("r10") = ctid; - stack = (intptr_t *)((long)((char *)stk + stksz) & -16); - *--stack = (long)arg; // push 1 - asm volatile("syscall" - : "=a"(ax) - : "0"(__NR_clone_linux), "D"(flags), "S"(stack), "d"(ptid), - "r"(r10), "r"(r8), "r"(r9) - : "rcx", "r11", "memory"); - if (ax > -4096u) { - errno = -ax; - return -1; - } - if (ax) return ax; - asm volatile("xor\t%%ebp,%%ebp\n\t" - "pop\t%%rdi\n\t" // pop 1 - "call\t*%0\n\t" - "xchg\t%%eax,%%edi\n\t" - "jmp\t_Exit1" - : /* no outputs */ - : "r"(r9) - : "memory"); - unreachable; -} - -/** - * Creates thread. - * - * This function follows the same ABI convention as the Linux userspace - * libraries, with a few small changes. The varargs has been removed to - * help prevent broken code, and the stack size and tls size parameters - * are introduced for compatibility with FreeBSD. - * - * @param func is your callback function - * @param stk points to the bottom of a caller allocated stack, which - * must be null when fork() and vfork() equivalent flags are used - * and furthermore this must be mmap()'d using MAP_STACK in order - * to work on OpenBSD - * @param stksz is the size of that stack in bytes which must be zero - * if the fork() or vfork() equivalent flags are used it's highly - * recommended that this value be GetStackSize(), or else kprintf - * and other runtime services providing memory safety can't do as - * good and quick of a job at that - * @param flags usually has one of - * - `SIGCHLD` will delegate to fork() - * - `CLONE_VFORK|CLONE_VM|SIGCHLD` means vfork() - * - `CLONE_VM|CLONE_FS|CLONE_FILES|CLONE_SIGHAND` for threads - * as part high bytes, and the low order byte may optionally contain - * a signal e.g. SIGCHLD, to enable parent notification on terminate - * although the signal isn't supported on non-Linux and non-NetBSD - * at the moment; 'flags' may optionally bitwise or the following: - * - `CLONE_PARENT_SETTID` is needed for `ctid` should be set - * - `CLONE_CHILD_SETTID` is needed for `ptid` should be set - * - `CLONE_SETTLS` is needed to set `%fs` segment to `tls` - * @param arg will be passed to your callback - * @param ptid lets the parent receive the child thread id; - * this parameter is ignored if `CLONE_PARENT_SETTID` is not set - * @param tls may be used to set the thread local storage segment; - * this parameter is ignored if `CLONE_SETTLS` is not set - * @param tlssz is the size of tls in bytes - * @param ctid lets the child receive its thread id; - * this parameter is ignored if `CLONE_CHILD_SETTID` is not set - * @return tid on success and 0 to the child, otherwise -1 w/ errno - * @returnstwice - * @threadsafe - */ -privileged int clone(int (*func)(void *), void *stk, size_t stksz, int flags, - void *arg, int *ptid, void *tls, size_t tlssz, int *ctid) { - int rc; - __threaded = true; - if (IsAsan() && - (!__asan_is_valid(stk, stksz) || - ((flags & CLONE_SETTLS) && !__asan_is_valid(tls, tlssz)) || - ((flags & CLONE_SETTLS) && !__asan_is_valid(tls, sizeof(long))) || - ((flags & CLONE_PARENT_SETTID) && - !__asan_is_valid(ptid, sizeof(*ptid))) || - ((flags & CLONE_CHILD_SETTID) && - !__asan_is_valid(ctid, sizeof(*ctid))))) { - rc = efault(); - } else if (IsLinux()) { - rc = CloneLinux(func, stk, stksz, flags, arg, ptid, tls, tlssz, ctid); - } else if (IsNetbsd()) { - rc = CloneNetbsd(func, stk, stksz, flags, arg, ptid, tls, tlssz, ctid); - } - - // polyfill fork() and vfork() use case on platforms w/o clone - else if ((SupportsWindows() || SupportsBsd()) && - flags == (CLONE_VFORK | CLONE_VM | SIGCHLD)) { - if (IsTiny()) { - rc = einval(); - } else if (!arg && !stksz) { - return vfork(); // don't log clone() - } else { - rc = einval(); - } - } else if ((SupportsWindows() || SupportsBsd()) && flags == SIGCHLD) { - if (IsTiny()) { - rc = eopnotsupp(); - } else if (!arg && !stksz) { - return fork(); // don't log clone() - } else { - rc = einval(); - } - } - - // we now assume we're creating a thread - // these platforms can't do signals the way linux does - else if ((flags & - ~(CLONE_SETTLS | CLONE_PARENT_SETTID | CLONE_CHILD_SETTID)) != - (CLONE_VM | CLONE_FS | CLONE_FILES | CLONE_SIGHAND)) { - rc = einval(); - } else if (IsFreebsd()) { - rc = CloneFreebsd(func, stk, stksz, flags, arg, ptid, tls, tlssz, ctid); - } else if (IsOpenbsd()) { - rc = CloneOpenbsd(func, stk, stksz, flags, arg, ptid, tls, tlssz, ctid); - } - - // These platforms can't do segment registers like linux does - else if (flags & CLONE_SETTLS) { - rc = einval(); - } else if (IsWindows()) { - rc = CloneWindows(func, stk, stksz, flags, arg, ptid, tls, tlssz, ctid); - } else { - rc = enosys(); - } - - STRACE("clone(%p, %p, %'zu, %#x, %p, %p, %p, %'zu, %p) → %d% m", func, stk, - stksz, flags, arg, ptid, tls, tlssz, ctid, rc); - return rc; -} diff --git a/libc/thread/freebsd.internal.h b/libc/thread/freebsd.internal.h index e92180099..32134c8ec 100644 --- a/libc/thread/freebsd.internal.h +++ b/libc/thread/freebsd.internal.h @@ -31,17 +31,6 @@ struct thr_param { struct rtprio *rtp; }; -static inline int thr_new(struct thr_param *param, int param_size) { - bool failed; - long ax, di, si; - asm volatile(CFLAG_ASM("syscall") - : CFLAG_CONSTRAINT(failed), "=a"(ax), "=D"(di), "=S"(si) - : "1"(455), "2"(param), "3"(param_size) - : "rcx", "rdx", "r8", "r9", "r10", "r11", "memory"); - if (failed) ax = -ax; - return ax; -} - COSMOPOLITAN_C_END_ #endif /* !(__ASSEMBLER__ + __LINKER__ + 0) */ #endif /* COSMOPOLITAN_LIBC_THREAD_FREEBSD_INTERNAL_H_ */ diff --git a/libc/thread/openbsd.internal.h b/libc/thread/openbsd.internal.h deleted file mode 100644 index 2c89f5d4f..000000000 --- a/libc/thread/openbsd.internal.h +++ /dev/null @@ -1,14 +0,0 @@ -#ifndef COSMOPOLITAN_LIBC_THREAD_OPENBSD_INTERNAL_H_ -#define COSMOPOLITAN_LIBC_THREAD_OPENBSD_INTERNAL_H_ -#if !(__ASSEMBLER__ + __LINKER__ + 0) -COSMOPOLITAN_C_START_ - -struct __tfork { - void *tf_tcb; - int32_t *tf_tid; - void *tf_stack; -}; - -COSMOPOLITAN_C_END_ -#endif /* !(__ASSEMBLER__ + __LINKER__ + 0) */ -#endif /* COSMOPOLITAN_LIBC_THREAD_OPENBSD_INTERNAL_H_ */ diff --git a/libc/thread/thread.mk b/libc/thread/thread.mk index 04d2c7818..1f99033d5 100644 --- a/libc/thread/thread.mk +++ b/libc/thread/thread.mk @@ -46,10 +46,9 @@ $(LIBC_THREAD_A).pkg: \ $(LIBC_THREAD_A_OBJS) \ $(foreach x,$(LIBC_THREAD_A_DIRECTDEPS),$($(x)_A).pkg) -# no red zone because asm("call") -o/$(MODE)/libc/thread/clone.o: \ +o/tinylinux/libc/thread/clone.o: \ OVERRIDE_CFLAGS += \ - -mno-red-zone + -ffunction-sections LIBC_THREAD_LIBS = $(foreach x,$(LIBC_THREAD_ARTIFACTS),$($(x))) LIBC_THREAD_SRCS = $(foreach x,$(LIBC_THREAD_ARTIFACTS),$($(x)_SRCS)) diff --git a/libc/thread/xnu.internal.h b/libc/thread/xnu.internal.h index 04b530704..ff23f8c8e 100644 --- a/libc/thread/xnu.internal.h +++ b/libc/thread/xnu.internal.h @@ -8,17 +8,21 @@ COSMOPOLITAN_C_START_ * @see darwin-libpthread/kern/kern_support.c */ -void *bsdthread_create(void *func, void *func_arg, void *stack, void *pthread, - uint32_t flags); +int bsdthread_create(void *func, void *func_arg, void *stack, void *pthread, + uint32_t flags); int bsdthread_terminate(void *stackaddr, size_t freesize, uint32_t port, uint32_t sem); -int bsdthread_register(void *threadstart, void *wqthread, uint32_t flags, - void *stack_addr_hint, void *targetconc_ptr, - uint32_t dispatchqueue_offset, uint32_t tsd_offset); +int bsdthread_register( + void (*threadstart)(void *pthread, int machport, void *(*func)(void *), + void *arg, intptr_t *, unsigned), + void (*wqthread)(void *pthread, void *machport, void *, void *, int), + uint32_t flags, void *stack_addr_hint, void *targetconc_ptr, + uint32_t dispatchqueue_offset, uint32_t tsd_offset); int bsdthread_ctl(void *cmd, void *arg1, void *arg2, void *arg3); uint64_t thread_selfid(void); uint64_t thread_selfusage(void); int thread_selfcounts(int type, void *buf, uint64_t nbytes); +int thread_fast_set_cthread_self(void *tls); COSMOPOLITAN_C_END_ #endif /* !(__ASSEMBLER__ + __LINKER__ + 0) */ diff --git a/test/libc/calls/tkill_test.c b/test/libc/calls/tkill_test.c new file mode 100644 index 000000000..e69de29bb diff --git a/test/libc/intrin/gettid_test.c b/test/libc/intrin/gettid_test.c new file mode 100644 index 000000000..e69de29bb diff --git a/test/libc/rand/rand64_test.c b/test/libc/rand/rand64_test.c index fd91be5b1..ba76907cc 100644 --- a/test/libc/rand/rand64_test.c +++ b/test/libc/rand/rand64_test.c @@ -33,6 +33,7 @@ #include "libc/sysv/consts/sa.h" #include "libc/sysv/consts/sig.h" #include "libc/testlib/testlib.h" +#include "libc/time/time.h" #define THREADS 8 #define ENTRIES 256 @@ -76,7 +77,6 @@ TEST(rand64, testThreadSafety_doesntProduceIdenticalValues) { sigset_t ss, oldss; void *stacks[THREADS]; int i, j, rc, ws, tid[THREADS]; - if (IsXnu()) return; struct sigaction oldsa; struct sigaction sa = {.sa_handler = OnChld, .sa_flags = SA_RESTART}; EXPECT_NE(-1, sigaction(SIGCHLD, &sa, &oldsa)); @@ -89,9 +89,9 @@ TEST(rand64, testThreadSafety_doesntProduceIdenticalValues) { } ready = false; for (i = 0; i < THREADS; ++i) { - stacks[i] = mmap(0, FRAMESIZE, PROT_READ | PROT_WRITE, - MAP_PRIVATE | MAP_ANONYMOUS | MAP_STACK, -1, 0); - tid[i] = clone(Thrasher, stacks[i], FRAMESIZE, + stacks[i] = mmap(0, GetStackSize(), PROT_READ | PROT_WRITE, + MAP_STACK | MAP_ANONYMOUS, -1, 0); + tid[i] = clone(Thrasher, stacks[i], GetStackSize(), CLONE_VM | CLONE_FS | CLONE_FILES | CLONE_SIGHAND, (void *)(intptr_t)i, 0, 0, 0, 0); ASSERT_NE(-1, tid[i]); @@ -110,6 +110,10 @@ TEST(rand64, testThreadSafety_doesntProduceIdenticalValues) { } } for (i = 0; i < THREADS; ++i) { - EXPECT_SYS(0, 0, munmap(stacks[i], FRAMESIZE)); + tkill(tid[i], SIGKILL); + errno = 0; + } + for (i = 0; i < THREADS; ++i) { + EXPECT_SYS(0, 0, munmap(stacks[i], GetStackSize())); } } diff --git a/test/libc/thread/clone_test.c b/test/libc/runtime/clone_test.c similarity index 80% rename from test/libc/thread/clone_test.c rename to test/libc/runtime/clone_test.c index 0566f8097..6cddceb71 100644 --- a/test/libc/thread/clone_test.c +++ b/test/libc/runtime/clone_test.c @@ -18,43 +18,52 @@ ╚─────────────────────────────────────────────────────────────────────────────*/ #include "libc/calls/calls.h" #include "libc/dce.h" +#include "libc/errno.h" #include "libc/intrin/spinlock.h" +#include "libc/runtime/stack.h" #include "libc/sysv/consts/clone.h" #include "libc/sysv/consts/map.h" #include "libc/sysv/consts/prot.h" +#include "libc/sysv/consts/sig.h" #include "libc/testlib/testlib.h" +#include "libc/time/time.h" -int x, thechilde; +char *stack; +int x, me, thechilde; _Alignas(64) volatile char lock; void SetUp(void) { x = 0; lock = 0; + me = gettid(); thechilde = 0; + ASSERT_NE(MAP_FAILED, (stack = mmap(0, GetStackSize(), PROT_READ | PROT_WRITE, + MAP_STACK | MAP_ANONYMOUS, -1, 0))); } -int thread(void *arg) { +void TearDown(void) { + tkill(thechilde, SIGKILL), errno = 0; + sched_yield(); + EXPECT_SYS(0, 0, munmap(stack, GetStackSize())); +} + +int CloneTest1(void *arg) { x = 42; ASSERT_EQ(23, (intptr_t)arg); thechilde = gettid(); + ASSERT_NE(gettid(), getpid()); _spunlock(&lock); return 0; } -TEST(clone, test) { - if (IsXnu()) return; - int me, tid; - char *stack; - me = gettid(); +TEST(clone, test1) { + int tid; _spinlock(&lock); - stack = mmap(0, FRAMESIZE, PROT_READ | PROT_WRITE, - MAP_PRIVATE | MAP_ANONYMOUS | MAP_STACK, -1, 0); - ASSERT_NE(-1, (tid = clone(thread, stack, FRAMESIZE, + ASSERT_NE(-1, (tid = clone(CloneTest1, stack, GetStackSize(), CLONE_VM | CLONE_FS | CLONE_FILES | CLONE_SIGHAND, (void *)23, 0, 0, 0, 0))); _spinlock(&lock); ASSERT_EQ(42, x); ASSERT_NE(me, tid); ASSERT_EQ(tid, thechilde); - EXPECT_SYS(0, 0, munmap(stack, FRAMESIZE)); }