From ec957491ead205cb36470a86adb75ae2b8464015 Mon Sep 17 00:00:00 2001 From: Justine Tunney Date: Sun, 20 Aug 2023 02:13:02 -0700 Subject: [PATCH] Make posix_spawn faster on Windows --- libc/calls/calls.mk | 2 + libc/calls/copy_file_range.c | 3 +- libc/calls/execve-nt.greg.c | 97 ++++++--- libc/calls/raise.c | 16 +- libc/calls/struct/fd.internal.h | 1 + libc/calls/{tkill.c => switchstacks.S} | 55 ++--- libc/calls/winstdin1.c | 4 +- libc/intrin/cosmo_once.c | 3 +- libc/intrin/createdirectory.c | 9 +- libc/stdio/posix_spawn.c | 286 +++++++++++++++++++++---- libc/stdio/posix_spawn_file_actions.c | 17 +- libc/thread/pthread_create.c | 20 ++ libc/thread/thread.h | 1 + libc/thread/tkill.c | 122 +++++++++++ tool/build/runitd.c | 1 + 15 files changed, 514 insertions(+), 123 deletions(-) rename libc/calls/{tkill.c => switchstacks.S} (57%) create mode 100644 libc/thread/tkill.c diff --git a/libc/calls/calls.mk b/libc/calls/calls.mk index 8bcb44b9c..9df4e65a7 100644 --- a/libc/calls/calls.mk +++ b/libc/calls/calls.mk @@ -205,6 +205,8 @@ o/$(MODE)/libc/calls/swapcontext.o: libc/calls/swapcontext.S @$(COMPILE) -AOBJECTIFY.S $(OBJECTIFY.S) $(OUTPUT_OPTION) -c $< o/$(MODE)/libc/calls/tailcontext.o: libc/calls/tailcontext.S @$(COMPILE) -AOBJECTIFY.S $(OBJECTIFY.S) $(OUTPUT_OPTION) -c $< +o/$(MODE)/libc/calls/switchstacks.o: libc/calls/switchstacks.S + @$(COMPILE) -AOBJECTIFY.S $(OBJECTIFY.S) $(OUTPUT_OPTION) -c $< LIBC_CALLS_LIBS = $(foreach x,$(LIBC_CALLS_ARTIFACTS),$($(x))) LIBC_CALLS_SRCS = $(foreach x,$(LIBC_CALLS_ARTIFACTS),$($(x)_SRCS)) diff --git a/libc/calls/copy_file_range.c b/libc/calls/copy_file_range.c index da4c8e945..b7983b080 100644 --- a/libc/calls/copy_file_range.c +++ b/libc/calls/copy_file_range.c @@ -16,6 +16,7 @@ │ TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR │ │ PERFORMANCE OF THIS SOFTWARE. │ ╚─────────────────────────────────────────────────────────────────────────────*/ +#include "libc/atomic.h" #include "libc/calls/blockcancel.internal.h" #include "libc/calls/calls.h" #include "libc/calls/cp.internal.h" @@ -33,7 +34,7 @@ #include "libc/sysv/errfuns.h" static struct CopyFileRange { - _Atomic(uint32_t) once; + atomic_uint once; bool ok; } g_copy_file_range; diff --git a/libc/calls/execve-nt.greg.c b/libc/calls/execve-nt.greg.c index 6c320c9ea..0f0ee1fbe 100644 --- a/libc/calls/execve-nt.greg.c +++ b/libc/calls/execve-nt.greg.c @@ -22,8 +22,10 @@ #include "libc/calls/ntspawn.h" #include "libc/calls/syscall-nt.internal.h" #include "libc/fmt/itoa.h" +#include "libc/intrin/kprintf.h" #include "libc/intrin/strace.internal.h" #include "libc/intrin/weaken.h" +#include "libc/macros.internal.h" #include "libc/mem/alloca.h" #include "libc/nt/accounting.h" #include "libc/nt/console.h" @@ -34,16 +36,22 @@ #include "libc/nt/struct/processinformation.h" #include "libc/nt/struct/startupinfo.h" #include "libc/nt/synchronization.h" +#include "libc/nt/thread.h" #include "libc/nt/thunk/msabi.h" #include "libc/runtime/memtrack.internal.h" #include "libc/runtime/runtime.h" +#include "libc/runtime/stack.h" #include "libc/sock/sock.h" #include "libc/str/str.h" #include "libc/sysv/consts/at.h" #include "libc/sysv/consts/map.h" #include "libc/sysv/consts/o.h" #include "libc/sysv/consts/ok.h" +#include "libc/sysv/consts/sig.h" #include "libc/sysv/errfuns.h" +#include "libc/thread/thread.h" + +#define keywords textwindows dontasan dontubsan dontinstrument extern long __klog_handle; @@ -51,18 +59,58 @@ __msabi extern typeof(CloseHandle) *const __imp_CloseHandle; __msabi extern typeof(WaitForSingleObject) *const __imp_WaitForSingleObject; __msabi extern typeof(GetExitCodeProcess) *const __imp_GetExitCodeProcess; __msabi extern typeof(UnmapViewOfFile) *const __imp_UnmapViewOfFile; +__msabi extern typeof(TerminateThread) *const __imp_TerminateThread; -static dontinstrument __msabi bool32 -BlockExecveConsoleEvent(uint32_t dwCtrlType) { - // block SIGINT and SIGQUIT in execve() parent process - return true; +wontreturn void __switch_stacks(intptr_t, long, long, long, + void (*)(intptr_t, intptr_t, long, long), + intptr_t); + +__msabi static keywords bool32 sys_execve_nt_event(uint32_t dwCtrlType) { + return true; // block sigint and sigquit in execve() parent process } -textwindows int sys_execve_nt(const char *program, char *const argv[], - char *const envp[]) { +static keywords void PurgeHandle(intptr_t h) { + if (h && h != -1) { + __imp_CloseHandle(h); + } +} + +static keywords void PurgeThread(intptr_t h) { + if (h && h != -1) { + __imp_TerminateThread(h, SIGKILL); + __imp_CloseHandle(h); + } +} + +// this function runs on the original tiny stack that windows gave us. +// we need to keep the original process alive simply to pass an int32. +// so we unmap all memory to avoid getting a double whammy after fork. +static keywords void sys_execve_nt_relay(intptr_t h, long b, long c, long d) { + uint32_t i, dwExitCode; + __imp_SetConsoleCtrlHandler((void *)sys_execve_nt_event, 1); + PurgeThread(g_fds.stdin.thread); + PurgeHandle(g_fds.stdin.reader); + PurgeHandle(g_fds.stdin.writer); + PurgeHandle(g_fds.p[0].handle); + PurgeHandle(g_fds.p[1].handle); + PurgeHandle(g_fds.p[2].handle); + for (i = 0; i < _mmi.i; ++i) { + __imp_UnmapViewOfFile((void *)((uintptr_t)_mmi.p[i].x << 16)); + PurgeHandle(_mmi.p[i].h); + } + do { + __imp_WaitForSingleObject(h, -1); + dwExitCode = kNtStillActive; + __imp_GetExitCodeProcess(h, &dwExitCode); + } while (dwExitCode == kNtStillActive); + __imp_ExitProcess(dwExitCode); + __builtin_unreachable(); +} + +keywords int sys_execve_nt(const char *program, char *const argv[], + char *const envp[]) { int rc; size_t i; - uint32_t dwExitCode; char progbuf[PATH_MAX]; struct NtStartupInfo startinfo; struct NtProcessInformation procinfo; @@ -89,16 +137,26 @@ textwindows int sys_execve_nt(const char *program, char *const argv[], ////////////////////////////////////////////////////////////////////////////// // execve operation is unrecoverable from this point + if (_weaken(pthread_kill_siblings_np)) { + _weaken(pthread_kill_siblings_np)(); + } + // close non-stdio and cloexec handles for (i = 0; i < g_fds.n; ++i) { if (g_fds.p[i].kind == kFdEmpty) { g_fds.p[i].handle = -1; } else if (i > 2 || (g_fds.p[i].flags & O_CLOEXEC)) { - __imp_CloseHandle(g_fds.p[i].handle); + PurgeHandle(g_fds.p[i].handle); g_fds.p[i].handle = -1; } } + if (_weaken(__klog_handle) && // + *_weaken(__klog_handle) != 0 && // + *_weaken(__klog_handle) != -1) { + PurgeHandle(*_weaken(__klog_handle)); + } + int bits; char buf[32], *v = 0; if (_weaken(socket)) { @@ -117,12 +175,6 @@ textwindows int sys_execve_nt(const char *program, char *const argv[], startinfo.hStdOutput = g_fds.p[1].handle; startinfo.hStdError = g_fds.p[2].handle; - if (_weaken(__klog_handle) && // - *_weaken(__klog_handle) != 0 && // - *_weaken(__klog_handle) != -1) { - __imp_CloseHandle(*_weaken(__klog_handle)); - } - // spawn the process rc = ntspawn(program, argv, envp, v, 0, 0, true, 0, 0, &startinfo, &procinfo); if (rc == -1) { @@ -131,19 +183,8 @@ textwindows int sys_execve_nt(const char *program, char *const argv[], } ////////////////////////////////////////////////////////////////////////////// - // zombie shell process remains, to wait for child and propagate its exit - // code + // zombify this process which lingers on to relay the status code - __imp_CloseHandle(g_fds.p[0].handle); - __imp_CloseHandle(g_fds.p[1].handle); - __imp_CloseHandle(procinfo.hThread); - __imp_SetConsoleCtrlHandler((void *)BlockExecveConsoleEvent, 1); - do { - __imp_WaitForSingleObject(procinfo.hProcess, -1); - dwExitCode = kNtStillActive; - __imp_GetExitCodeProcess(procinfo.hProcess, &dwExitCode); - } while (dwExitCode == kNtStillActive); - __imp_CloseHandle(procinfo.hProcess); - __imp_ExitProcess(dwExitCode); - notpossible; + PurgeHandle(procinfo.hThread); + __switch_stacks(procinfo.hProcess, 0, 0, 0, sys_execve_nt_relay, __oldstack); } diff --git a/libc/calls/raise.c b/libc/calls/raise.c index e7a85f7f2..1c41b8a24 100644 --- a/libc/calls/raise.c +++ b/libc/calls/raise.c @@ -21,6 +21,8 @@ #include "libc/calls/syscall-sysv.internal.h" #include "libc/dce.h" #include "libc/intrin/strace.internal.h" +#include "libc/intrin/weaken.h" +#include "libc/nt/runtime.h" #include "libc/runtime/internal.h" #include "libc/sysv/consts/sicode.h" #include "libc/sysv/consts/sig.h" @@ -62,10 +64,20 @@ int raise(int sig) { RaiseSigFpe(); rc = 0; #endif - } else if (!IsWindows() && !IsMetal()) { + } else if (IsLinux() || IsXnu() || IsFreebsd() || IsOpenbsd() || IsNetbsd()) { rc = sys_tkill(gettid(), sig, 0); + } else if (IsWindows() || IsMetal()) { + if (IsWindows() && sig == SIGKILL) { + // TODO(jart): Isn't this implemented by __sig_raise()? + if (_weaken(__restore_console_win32)) { + _weaken(__restore_console_win32)(); + } + ExitProcess(sig); + } else { + rc = __sig_raise(sig, SI_TKILL); + } } else { - rc = __sig_raise(sig, SI_TKILL); + __builtin_unreachable(); } STRACE("...raise(%G) → %d% m", sig, rc); return rc; diff --git a/libc/calls/struct/fd.internal.h b/libc/calls/struct/fd.internal.h index 87eb0fc10..69aff8a4f 100644 --- a/libc/calls/struct/fd.internal.h +++ b/libc/calls/struct/fd.internal.h @@ -41,6 +41,7 @@ struct StdinRelay { int64_t handle; /* should == g_fds.p[0].handle */ int64_t reader; /* ReadFile() use this instead */ int64_t writer; /* only used by WinStdinThread */ + int64_t thread; /* handle for the stdio thread */ }; struct Fds { diff --git a/libc/calls/tkill.c b/libc/calls/switchstacks.S similarity index 57% rename from libc/calls/tkill.c rename to libc/calls/switchstacks.S index d767aa75d..681f48415 100644 --- a/libc/calls/tkill.c +++ b/libc/calls/switchstacks.S @@ -1,7 +1,7 @@ -/*-*- mode:c;indent-tabs-mode:nil;c-basic-offset:2;tab-width:8;coding:utf-8 -*-│ -│vi: set net ft=c ts=2 sts=2 sw=2 fenc=utf-8 :vi│ +/*-*- mode:unix-assembly; indent-tabs-mode:t; tab-width:8; coding:utf-8 -*-│ +│vi: set et ft=asm ts=8 tw=8 fenc=utf-8 :vi│ ╞══════════════════════════════════════════════════════════════════════════════╡ -│ Copyright 2022 Justine Alexandra Roberts Tunney │ +│ Copyright 2023 Justine Alexandra Roberts Tunney │ │ │ │ Permission to use, copy, modify, and/or distribute this software for │ │ any purpose with or without fee is hereby granted, provided that the │ @@ -16,38 +16,19 @@ │ TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR │ │ PERFORMANCE OF THIS SOFTWARE. │ ╚─────────────────────────────────────────────────────────────────────────────*/ -#include "libc/calls/calls.h" -#include "libc/calls/sig.internal.h" -#include "libc/calls/syscall-sysv.internal.h" -#include "libc/calls/syscall_support-sysv.internal.h" -#include "libc/dce.h" -#include "libc/intrin/strace.internal.h" -#include "libc/sysv/consts/sicode.h" +#include "libc/macros.internal.h" -// OpenBSD has an optional `tib` parameter for extra safety -int __tkill(int tid, int sig, void *tib) { - int rc; - if (!IsWindows() && !IsMetal()) { - rc = sys_tkill(tid, sig, tib); - } else { - rc = __sig_add(tid, sig, SI_TKILL); - } - STRACE("tkill(%d, %G) → %d% m", tid, sig, rc); - return rc; -} - -/** - * Kills thread. - * - * @param tid is thread id - * @param sig does nothing on xnu - * @return 0 on success, or -1 w/ errno - * @raise EAGAIN if `RLIMIT_SIGPENDING` was exceeded - * @raise EINVAL if `tid` or `sig` were invalid - * @raise ESRCH if no such `tid` existed - * @raise EPERM if permission was denied - * @asyncsignalsafe - */ -int tkill(int tid, int sig) { - return __tkill(tid, sig, 0); -} +__switch_stacks: +#ifdef __x86_64__ + mov %r9,%rsp + and $-16,%rsp + xor %rbp,%rbp + call *%r8 +#elif defined(__aarch64__) + and sp,x5,#-16 + mov x29,0 + blr x4 +#else +#error "unsupported architecture" +#endif + .endfn __switch_stacks,globl diff --git a/libc/calls/winstdin1.c b/libc/calls/winstdin1.c index bc8442b0a..18766d1f0 100644 --- a/libc/calls/winstdin1.c +++ b/libc/calls/winstdin1.c @@ -135,8 +135,8 @@ dontasan dontubsan dontinstrument textwindows void WinMainStdin(void) { NTTRACE(" CreateThread failed"); return; } - __imp_CloseHandle(hThread); - g_fds.stdin.handle = hStdin; + g_fds.stdin.thread = hStdin; + g_fds.stdin.handle = hThread; g_fds.stdin.reader = hReader; g_fds.stdin.writer = hWriter; g_fds.stdin.inisem = hSemaphore; diff --git a/libc/intrin/cosmo_once.c b/libc/intrin/cosmo_once.c index 4ff1e87e2..a4efb5825 100644 --- a/libc/intrin/cosmo_once.c +++ b/libc/intrin/cosmo_once.c @@ -16,6 +16,7 @@ │ TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR │ │ PERFORMANCE OF THIS SOFTWARE. │ ╚─────────────────────────────────────────────────────────────────────────────*/ +#include "libc/atomic.h" #include "libc/cosmo.h" #include "libc/errno.h" #include "libc/intrin/atomic.h" @@ -35,7 +36,7 @@ * * @return 0 on success, or errno on error */ -errno_t cosmo_once(_Atomic(uint32_t) *once, void init(void)) { +errno_t cosmo_once(atomic_uint *once, void init(void)) { uint32_t old; switch ((old = atomic_load_explicit(once, memory_order_relaxed))) { case INIT: diff --git a/libc/intrin/createdirectory.c b/libc/intrin/createdirectory.c index 60da1ac9c..efe57c455 100644 --- a/libc/intrin/createdirectory.c +++ b/libc/intrin/createdirectory.c @@ -30,13 +30,12 @@ __msabi extern typeof(CreateDirectory) *const __imp_CreateDirectoryW; * @return handle, or -1 on failure * @note this wrapper takes care of ABI, STRACE(), and __winerr() */ -textwindows bool32 -CreateDirectory(const char16_t *lpPathName, - struct NtSecurityAttributes *lpSecurityAttributes) { +textwindows bool32 CreateDirectory(const char16_t *lpPathName, + struct NtSecurityAttributes *lpSecurity) { bool32 ok; - ok = __imp_CreateDirectoryW(lpPathName, lpSecurityAttributes); + ok = __imp_CreateDirectoryW(lpPathName, lpSecurity); if (!ok) __winerr(); NTTRACE("CreateDirectory(%#hs, %s) → %hhhd% m", lpPathName, - DescribeNtSecurityAttributes(lpSecurityAttributes), ok); + DescribeNtSecurityAttributes(lpSecurity), ok); return ok; } diff --git a/libc/stdio/posix_spawn.c b/libc/stdio/posix_spawn.c index 0c6afe48f..ac2b19d13 100644 --- a/libc/stdio/posix_spawn.c +++ b/libc/stdio/posix_spawn.c @@ -17,50 +17,253 @@ │ PERFORMANCE OF THIS SOFTWARE. │ ╚─────────────────────────────────────────────────────────────────────────────*/ #include "libc/stdio/posix_spawn.h" +#include "libc/assert.h" #include "libc/calls/calls.h" +#include "libc/calls/internal.h" +#include "libc/calls/ntspawn.h" +#include "libc/calls/struct/fd.internal.h" #include "libc/calls/struct/sigaction.h" #include "libc/calls/struct/sigset.h" +#include "libc/dce.h" #include "libc/errno.h" +#include "libc/fmt/itoa.h" +#include "libc/fmt/magnumstrs.internal.h" +#include "libc/intrin/asan.internal.h" +#include "libc/intrin/describeflags.internal.h" #include "libc/intrin/strace.internal.h" #include "libc/intrin/weaken.h" +#include "libc/mem/alloca.h" +#include "libc/nt/enum/processcreationflags.h" +#include "libc/nt/enum/startf.h" +#include "libc/nt/runtime.h" +#include "libc/nt/struct/processinformation.h" +#include "libc/nt/struct/startupinfo.h" #include "libc/runtime/runtime.h" +#include "libc/sock/sock.h" +#include "libc/stdio/posix_spawn.h" #include "libc/stdio/posix_spawn.internal.h" +#include "libc/str/str.h" +#include "libc/sysv/consts/o.h" #include "libc/sysv/consts/sig.h" #include "libc/thread/thread.h" #include "libc/thread/tls.h" -static int RunFileActions(struct _posix_faction *a) { - int t; - if (!a) return 0; - if (RunFileActions(a->next) == -1) return -1; - switch (a->action) { - case _POSIX_SPAWN_CLOSE: - return close(a->fildes); - case _POSIX_SPAWN_DUP2: - return dup2(a->fildes, a->newfildes); - case _POSIX_SPAWN_OPEN: - if ((t = open(a->path, a->oflag, a->mode)) == -1) return -1; - if (t != a->fildes) { - if (dup2(t, a->fildes) == -1) { - close(t); - return -1; - } - if (close(t) == -1) { - return -1; - } +static void posix_spawn_cleanup3fds(int fds[3]) { + for (int i = 0; i < 3; ++i) { + if (fds[i] != -1) { + int e = errno; + if (close(fds[i])) { + errno = e; } - return 0; - default: - __builtin_unreachable(); + } + } +} + +static const char *DescribePid(char buf[12], int err, int *pid) { + if (err) return "n/a"; + if (!pid) return "NULL"; + FormatInt32(buf, *pid); + return buf; +} + +static textwindows errno_t posix_spawn_windows_impl( + int *pid, const char *path, const posix_spawn_file_actions_t *file_actions, + const posix_spawnattr_t *attrp, char *const argv[], char *const envp[]) { + int i; + + // create file descriptor work area + char stdio_kind[3] = {kFdEmpty, kFdEmpty, kFdEmpty}; + intptr_t stdio_handle[3] = {-1, -1, -1}; + int close_this_fd_later[3] = {-1, -1, -1}; + for (i = 0; i < 3; ++i) { + if (g_fds.p[i].kind != kFdEmpty && !(g_fds.p[i].flags & O_CLOEXEC)) { + stdio_kind[i] = g_fds.p[i].kind; + stdio_handle[i] = g_fds.p[i].handle; + } + } + + // reserve a fake pid for this spawn + int child = __reservefd(-1); + + // apply user file actions + if (file_actions) { + int err = 0; + for (struct _posix_faction *a = *file_actions; a && !err; a = a->next) { + switch (a->action) { + case _POSIX_SPAWN_CLOSE: + unassert(a->fildes < 3u); + stdio_kind[a->fildes] = kFdEmpty; + stdio_handle[a->fildes] = -1; + break; + case _POSIX_SPAWN_DUP2: + unassert(a->newfildes < 3u); + if (__isfdopen(a->fildes)) { + stdio_kind[a->newfildes] = g_fds.p[a->fildes].kind; + stdio_handle[a->newfildes] = g_fds.p[a->fildes].handle; + } else { + err = EBADF; + } + break; + case _POSIX_SPAWN_OPEN: { + int fd, e = errno; + unassert(a->fildes < 3u); + if ((fd = open(a->path, a->oflag, a->mode)) != -1) { + close_this_fd_later[a->fildes] = fd; + stdio_kind[a->fildes] = g_fds.p[fd].kind; + stdio_handle[a->fildes] = g_fds.p[fd].handle; + } else { + err = errno; + errno = e; + } + break; + } + default: + __builtin_unreachable(); + } + } + if (err) { + posix_spawn_cleanup3fds(close_this_fd_later); + __releasefd(child); + return err; + } + } + + // create the windows process start info + int bits; + char buf[32], *v = 0; + if (_weaken(socket)) { + for (bits = i = 0; i < 3; ++i) { + if (stdio_kind[i] == kFdSocket) { + bits |= 1 << i; + } + } + FormatInt32(stpcpy(buf, "__STDIO_SOCKETS="), bits); + v = buf; + } + struct NtStartupInfo startinfo = { + .cb = sizeof(struct NtStartupInfo), + .dwFlags = kNtStartfUsestdhandles, + .hStdInput = stdio_handle[0], + .hStdOutput = stdio_handle[1], + .hStdError = stdio_handle[2], + }; + + // figure out the flags + short flags; + bool bInheritHandles = false; + uint32_t dwCreationFlags = 0; + for (i = 0; i < 3; ++i) { + bInheritHandles |= stdio_handle[i] != -1; + } + if (attrp && *attrp && !posix_spawnattr_getflags(attrp, &flags)) { + if (flags & POSIX_SPAWN_SETSID) { + dwCreationFlags |= kNtDetachedProcess; + } + if (flags & POSIX_SPAWN_SETPGROUP) { + dwCreationFlags |= kNtCreateNewProcessGroup; + } + } + + // launch the process + int rc, e = errno; + struct NtProcessInformation procinfo; + if (!envp) envp = environ; + rc = ntspawn(path, argv, envp, v, 0, 0, bInheritHandles, dwCreationFlags, 0, + &startinfo, &procinfo); + posix_spawn_cleanup3fds(close_this_fd_later); + if (rc == -1) { + int err = errno; + __releasefd(child); + errno = e; + return err; + } + + // track the process + CloseHandle(procinfo.hThread); + g_fds.p[child].kind = kFdProcess; + g_fds.p[child].handle = procinfo.hProcess; + g_fds.p[child].flags = O_CLOEXEC; + g_fds.p[child].zombie = false; + + // return the result + if (pid) *pid = child; + return 0; +} + +static textwindows dontinline errno_t posix_spawn_windows( + int *pid, const char *path, const posix_spawn_file_actions_t *file_actions, + const posix_spawnattr_t *attrp, char *const argv[], char *const envp[]) { + int err; + if (!path || !argv || + (IsAsan() && (!__asan_is_valid_str(path) || // + !__asan_is_valid_strlist(argv) || // + (envp && !__asan_is_valid_strlist(envp))))) { + err = EFAULT; + } else { + err = posix_spawn_windows_impl(pid, path, file_actions, attrp, argv, envp); + } + STRACE("posix_spawn([%s], %#s, %s, %s) → %s", + DescribePid(alloca(12), err, pid), path, DescribeStringList(argv), + DescribeStringList(envp), !err ? "0" : _strerrno(err)); + return err; +} + +static wontreturn void posix_spawn_die(const char *fail_func) { + STRACE("posix_spawn: %s failed% m", fail_func); + _Exit(127); +} + +static void RunUnixFileActions(struct _posix_faction *a) { + for (; a; a = a->next) { + switch (a->action) { + case _POSIX_SPAWN_CLOSE: + if (close(a->fildes)) { + posix_spawn_die("close"); + } + break; + case _POSIX_SPAWN_DUP2: + if (dup2(a->fildes, a->newfildes) == -1) { + posix_spawn_die("dup2"); + } + break; + case _POSIX_SPAWN_OPEN: { + int t; + if ((t = open(a->path, a->oflag, a->mode)) == -1) { + posix_spawn_die("open"); + } + if (t != a->fildes) { + if (dup2(t, a->fildes) == -1) { + close(t); + posix_spawn_die("dup2"); + } + if (close(t)) { + posix_spawn_die("close"); + } + } + break; + } + default: + __builtin_unreachable(); + } } } /** * Spawns process, the POSIX way. * - * This function provides an API for vfork() that's intended to be less - * terrifying to the uninitiated, since it only lets you define actions - * that are @vforksafe. This function requires TLS not be disabled. + * This provides superior process creation performance across systems. + * + * Processes are normally spawned by calling fork() and execve(), but + * that goes slow on Windows if the caller has allocated a nontrivial + * number of memory mappings, all of which need to be copied into the + * forked child, only to be destroyed a moment later. On UNIX systems + * fork() bears a similar cost that's 100x less bad, which is copying + * the page tables. So what this implementation does is on Windows it + * calls CreateProcess() directly and on UNIX it uses vfork() if it's + * possible (XNU and OpenBSD don't have it). + * + * If the child process exits with status 127 then use the `--strace` + * flag to get an explanation of failures that occurred during spawn. * * @param pid if non-null shall be set to child pid on success * @param path is resolved path of program which is not `$PATH` searched @@ -82,19 +285,20 @@ errno_t posix_spawn(int *pid, const char *path, int s, child, policy; struct sched_param param; struct sigaction dfl = {0}; + if (IsWindows()) { + return posix_spawn_windows(pid, path, file_actions, attrp, argv, envp); + } if (!(child = vfork())) { if (attrp && *attrp) { posix_spawnattr_getflags(attrp, &flags); if (flags & POSIX_SPAWN_SETSID) { if (setsid()) { - STRACE("posix_spawn fail #%d% m", 1); - _Exit(127); + posix_spawn_die("setsid"); } } if (flags & POSIX_SPAWN_SETPGROUP) { if (setpgid(0, (*attrp)->pgroup)) { - STRACE("posix_spawn fail #%d% m", 2); - _Exit(127); + posix_spawn_die("setpgid"); } } if (flags & POSIX_SPAWN_SETSIGMASK) { @@ -103,47 +307,39 @@ errno_t posix_spawn(int *pid, const char *path, } if ((flags & POSIX_SPAWN_RESETIDS) && (setgid(getgid()) || setuid(getuid()))) { - STRACE("posix_spawn fail #%d% m", 3); - _Exit(127); + posix_spawn_die("setuid"); } if (flags & POSIX_SPAWN_SETSIGDEF) { for (s = 1; s < 32; s++) { if (sigismember(&(*attrp)->sigdefault, s)) { - if (sigaction(s, &dfl, 0) == -1) { - STRACE("posix_spawn fail #%d% m", 4); - _Exit(127); + if (sigaction(s, &dfl, 0)) { + posix_spawn_die("sigaction"); } } } } } if (file_actions) { - if (RunFileActions(*file_actions) == -1) { - STRACE("posix_spawn fail #%d% m", 5); - _Exit(127); - } + RunUnixFileActions(*file_actions); } if (attrp && *attrp) { if (flags & POSIX_SPAWN_SETSCHEDULER) { posix_spawnattr_getschedpolicy(attrp, &policy); posix_spawnattr_getschedparam(attrp, ¶m); if (sched_setscheduler(0, policy, ¶m) == -1) { - STRACE("posix_spawn fail #%d% m", 6); - _Exit(127); + posix_spawn_die("sched_setscheduler"); } } if (flags & POSIX_SPAWN_SETSCHEDPARAM) { posix_spawnattr_getschedparam(attrp, ¶m); - if (sched_setparam(0, ¶m) == -1) { - STRACE("posix_spawn fail #%d% m", 7); - _Exit(127); + if (sched_setparam(0, ¶m)) { + posix_spawn_die("sched_setparam"); } } } if (!envp) envp = environ; execve(path, argv, envp); - STRACE("posix_spawn fail #%d% m", 8); - _Exit(127); + posix_spawn_die("execve"); } else if (child != -1) { if (pid) *pid = child; return 0; diff --git a/libc/stdio/posix_spawn_file_actions.c b/libc/stdio/posix_spawn_file_actions.c index 2efe41dad..e7a061bdb 100644 --- a/libc/stdio/posix_spawn_file_actions.c +++ b/libc/stdio/posix_spawn_file_actions.c @@ -16,6 +16,7 @@ │ TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR │ │ PERFORMANCE OF THIS SOFTWARE. │ ╚─────────────────────────────────────────────────────────────────────────────*/ +#include "libc/dce.h" #include "libc/errno.h" #include "libc/fmt/fmt.h" #include "libc/mem/mem.h" @@ -27,8 +28,8 @@ static int AddFileAction(posix_spawn_file_actions_t *l, struct _posix_faction a) { struct _posix_faction *ap; if (!(ap = malloc(sizeof(*ap)))) return ENOMEM; - a.next = *l; *ap = a; + while (*l) l = &(*l)->next; *l = ap; return 0; } @@ -36,6 +37,8 @@ static int AddFileAction(posix_spawn_file_actions_t *l, /** * Initializes posix_spawn() file actions list. * + * File actions get applied in the same order as they're registered. + * * @param file_actions will need posix_spawn_file_actions_destroy() * @return 0 on success, or errno on error */ @@ -68,9 +71,12 @@ int posix_spawn_file_actions_destroy(posix_spawn_file_actions_t *file_actions) { * @param file_actions was initialized by posix_spawn_file_actions_init() * @return 0 on success, or errno on error * @raise ENOMEM if we require more vespene gas + * @raise EBADF if `fildes` is negative */ int posix_spawn_file_actions_addclose(posix_spawn_file_actions_t *file_actions, int fildes) { + if (fildes < 0) return EBADF; + if (IsWindows() && fildes > 2) return 0; return AddFileAction(file_actions, (struct _posix_faction){ .action = _POSIX_SPAWN_CLOSE, .fildes = fildes, @@ -83,9 +89,13 @@ int posix_spawn_file_actions_addclose(posix_spawn_file_actions_t *file_actions, * @param file_actions was initialized by posix_spawn_file_actions_init() * @return 0 on success, or errno on error * @raise ENOMEM if we require more vespene gas + * @raise EBADF if 'fildes' or `newfildes` is negative + * @raise ENOTSUP if `newfildes` isn't 0, 1, or 2 on Windows */ int posix_spawn_file_actions_adddup2(posix_spawn_file_actions_t *file_actions, int fildes, int newfildes) { + if (fildes < 0 || newfildes < 0) return EBADF; + if (IsWindows() && newfildes > 2) return ENOTSUP; return AddFileAction(file_actions, (struct _posix_faction){ .action = _POSIX_SPAWN_DUP2, .fildes = fildes, @@ -97,15 +107,18 @@ int posix_spawn_file_actions_adddup2(posix_spawn_file_actions_t *file_actions, * Add an open action to object. * * @param file_actions was initialized by posix_spawn_file_actions_init() - * @param filedes is what open() result gets duplicated to + * @param fildes is what open() result gets duplicated to * @param path will be safely copied * @return 0 on success, or errno on error * @raise ENOMEM if we require more vespene gas + * @raise EBADF if `fildes` is negative + * @raise ENOTSUP if `fildes` isn't 0, 1, or 2 on Windows */ int posix_spawn_file_actions_addopen(posix_spawn_file_actions_t *file_actions, int fildes, const char *path, int oflag, unsigned mode) { if (fildes < 0) return EBADF; + if (IsWindows() && fildes > 2) return ENOTSUP; if (!(path = strdup(path))) return ENOMEM; return AddFileAction(file_actions, (struct _posix_faction){ .action = _POSIX_SPAWN_OPEN, diff --git a/libc/thread/pthread_create.c b/libc/thread/pthread_create.c index f4736e8d5..5c031edba 100644 --- a/libc/thread/pthread_create.c +++ b/libc/thread/pthread_create.c @@ -71,6 +71,26 @@ void _pthread_free(struct PosixThread *pt) { free(pt); } +void pthread_kill_siblings_np(void) { + struct Dll *e, *e2; + struct PosixThread *pt, *self; + enum PosixThreadStatus status; + self = (struct PosixThread *)__get_tls()->tib_pthread; + pthread_spin_lock(&_pthread_lock); + for (e = dll_first(_pthread_list); e; e = e2) { + e2 = dll_next(_pthread_list, e); + pt = POSIXTHREAD_CONTAINER(e); + if (pt != self) { + status = atomic_load_explicit(&pt->status, memory_order_acquire); + pthread_kill((pthread_t)pt, SIGKILL); + dll_remove(&_pthread_list, e); + pthread_spin_unlock(&_pthread_lock); + _pthread_free(pt); + } + } + pthread_spin_unlock(&_pthread_lock); +} + static int PosixThread(void *arg, int tid) { void *rc; struct sigaltstack ss; diff --git a/libc/thread/thread.h b/libc/thread/thread.h index 8752f5a12..482d52222 100644 --- a/libc/thread/thread.h +++ b/libc/thread/thread.h @@ -194,6 +194,7 @@ int pthread_spin_unlock(pthread_spinlock_t *) paramsnonnull(); int pthread_testcancel_np(void); int pthread_tryjoin_np(pthread_t, void **); int pthread_yield(void); +void pthread_kill_siblings_np(void); pthread_id_np_t pthread_getthreadid_np(void); pthread_t pthread_self(void) pureconst; void *pthread_getspecific(pthread_key_t); diff --git a/libc/thread/tkill.c b/libc/thread/tkill.c new file mode 100644 index 000000000..85067bb78 --- /dev/null +++ b/libc/thread/tkill.c @@ -0,0 +1,122 @@ +/*-*- mode:c;indent-tabs-mode:nil;c-basic-offset:2;tab-width:8;coding:utf-8 -*-│ +│vi: set net ft=c ts=2 sts=2 sw=2 fenc=utf-8 :vi│ +╞══════════════════════════════════════════════════════════════════════════════╡ +│ Copyright 2022 Justine Alexandra Roberts Tunney │ +│ │ +│ Permission to use, copy, modify, and/or distribute this software for │ +│ any purpose with or without fee is hereby granted, provided that the │ +│ above copyright notice and this permission notice appear in all copies. │ +│ │ +│ THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL │ +│ WARRANTIES WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED │ +│ WARRANTIES OF MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE │ +│ AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL │ +│ DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR │ +│ PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER │ +│ TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR │ +│ PERFORMANCE OF THIS SOFTWARE. │ +╚─────────────────────────────────────────────────────────────────────────────*/ +#include "libc/calls/calls.h" +#include "libc/calls/sig.internal.h" +#include "libc/calls/syscall-sysv.internal.h" +#include "libc/calls/syscall_support-nt.internal.h" +#include "libc/dce.h" +#include "libc/errno.h" +#include "libc/intrin/atomic.h" +#include "libc/intrin/dll.h" +#include "libc/intrin/strace.internal.h" +#include "libc/nt/enum/threadaccess.h" +#include "libc/nt/runtime.h" +#include "libc/nt/thread.h" +#include "libc/sysv/consts/sicode.h" +#include "libc/sysv/consts/sig.h" +#include "libc/sysv/errfuns.h" +#include "libc/thread/posixthread.internal.h" +#include "libc/thread/thread.h" +#include "libc/thread/tls.h" + +static dontinline textwindows int __tkill_nt(int tid, int sig, + struct CosmoTib *tib) { + + // check to see if this is a cosmo posix thread + int rc = 0; + struct Dll *e; + bool found = false; + pthread_spin_lock(&_pthread_lock); + for (e = dll_first(_pthread_list); e; e = dll_next(_pthread_list, e)) { + enum PosixThreadStatus status; + struct PosixThread *pt = POSIXTHREAD_CONTAINER(e); + if (tib && tib != pt->tib) continue; + int other = atomic_load_explicit(&pt->tib->tib_tid, memory_order_acquire); + if (!other || tid != other) continue; + status = atomic_load_explicit(&pt->status, memory_order_acquire); + found = true; + if (status < kPosixThreadTerminated) { + if (sig == SIGKILL) { + intptr_t h; + if ((h = OpenThread(kNtThreadTerminate, false, tid))) { + TerminateThread(h, sig); + CloseHandle(h); + } + atomic_store_explicit(&pt->status, kPosixThreadTerminated, + memory_order_release); + } else { + rc = __sig_add(tid, sig, SI_TKILL); + } + } else { + // already dead but not joined + } + break; + } + pthread_spin_unlock(&_pthread_lock); + if (found) { + return rc; + } + + // otherwise try our luck sigkilling a manually made thread + if (!tib) { + intptr_t h; + if ((h = OpenThread(kNtThreadTerminate, false, tid))) { + if (TerminateThread(h, sig)) { + return 0; + } else { + return __winerr(); + } + CloseHandle(h); + } else { + return __winerr(); + } + } else { + return esrch(); + } +} + +// OpenBSD has an optional `tib` parameter for extra safety. +int __tkill(int tid, int sig, void *tib) { + int rc; + if (IsLinux() || IsXnu() || IsFreebsd() || IsOpenbsd() || IsNetbsd()) { + rc = sys_tkill(tid, sig, tib); + } else if (IsWindows()) { + rc = __tkill_nt(tid, sig, tib); + } else { + rc = enosys(); + } + STRACE("tkill(%d, %G) → %d% m", tid, sig, rc); + return rc; +} + +/** + * Kills thread. + * + * @param tid is thread id + * @param sig does nothing on xnu + * @return 0 on success, or -1 w/ errno + * @raise EAGAIN if `RLIMIT_SIGPENDING` was exceeded + * @raise EINVAL if `tid` or `sig` were invalid + * @raise ESRCH if no such `tid` existed + * @raise EPERM if permission was denied + * @asyncsignalsafe + */ +int tkill(int tid, int sig) { + return __tkill(tid, sig, 0); +} diff --git a/tool/build/runitd.c b/tool/build/runitd.c index a3c5973ef..e8062e706 100644 --- a/tool/build/runitd.c +++ b/tool/build/runitd.c @@ -30,6 +30,7 @@ #include "libc/log/log.h" #include "libc/macros.internal.h" #include "libc/mem/gc.h" +#include "libc/mem/mem.h" #include "libc/nexgen32e/crc32.h" #include "libc/runtime/runtime.h" #include "libc/sock/sock.h"