diff --git a/libc/calls/diagnose_syscall.S b/libc/calls/diagnose_syscall.S new file mode 100644 index 000000000..b8319e944 --- /dev/null +++ b/libc/calls/diagnose_syscall.S @@ -0,0 +1,110 @@ +/*-*- mode:unix-assembly; indent-tabs-mode:t; tab-width:8; coding:utf-8 -*-│ +│vi: set et ft=asm ts=8 tw=8 fenc=utf-8 :vi│ +╞══════════════════════════════════════════════════════════════════════════════╡ +│ Copyright 2022 Justine Alexandra Roberts Tunney │ +│ │ +│ Permission to use, copy, modify, and/or distribute this software for │ +│ any purpose with or without fee is hereby granted, provided that the │ +│ above copyright notice and this permission notice appear in all copies. │ +│ │ +│ THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL │ +│ WARRANTIES WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED │ +│ WARRANTIES OF MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE │ +│ AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL │ +│ DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR │ +│ PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER │ +│ TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR │ +│ PERFORMANCE OF THIS SOFTWARE. │ +╚─────────────────────────────────────────────────────────────────────────────*/ +#include "libc/macros.internal.h" +.privileged + +diagnose_syscall: + push %rbp + mov %rsp,%rbp + push %rbx + push %r12 + push %r13 + push %r14 + push %r15 + + mov $0x7fffffff,%eax + add $4,%eax # set sf/of/pf + + mov %rdi,%rax # nr + mov %rsi,%rdi # arg 1 + mov %rdx,%rsi # arg 2 + mov %rcx,%rdx # arg 3 + mov %r8,%r10 # arg 4 + mov %r9,%r8 # arg 5 + mov 16(%rbp),%r9 # arg 6 + push 24(%rbp) # arg 7 + push %rax # fake ret addr + mov 32(%rbp),%r12 # ucontext before + mov 40(%rbp),%r13 # ucontext after + xor %ecx,%ecx + xor %r11d,%r11d + mov $0x5555555555555555,%r11 + mov $0x5555555555555555,%r14 + mov $0x5555555555555555,%r15 + mov $0x5555555555555555,%rbx + +// save machine state before system call + pushf + pop 176(%r12) + mov %r8,40(%r12) + mov %r9,48(%r12) + mov %r10,56(%r12) + mov %r11,64(%r12) + mov %r12,72(%r12) + mov %r13,80(%r12) + mov %r14,88(%r12) + mov %r15,96(%r12) + mov %rdi,104(%r12) + mov %rsi,112(%r12) + mov %rbp,120(%r12) + mov %rbx,128(%r12) + mov %rdx,136(%r12) + mov %rax,144(%r12) + mov %rcx,152(%r12) + push %rbx + lea 320(%r12),%rbx + mov %rbx,224(%r12) # set fpregs ptr + pop %rbx + + syscall + +// save machine state after system call + pushf + pop 176(%r13) + mov %r8,40(%r13) + mov %r9,48(%r13) + mov %r10,56(%r13) + mov %r11,64(%r13) + mov %r12,72(%r13) + mov %r13,80(%r13) + mov %r14,88(%r13) + mov %r15,96(%r13) + mov %rdi,104(%r13) + mov %rsi,112(%r13) + mov %rbp,120(%r13) + mov %rbx,128(%r13) + mov %rdx,136(%r13) + mov %rax,144(%r13) + mov %rcx,152(%r13) + push %rbx + lea 320(%r13),%rbx + mov %rbx,224(%r13) # set fpregs ptr + pop %rbx + + pop %r13 + pop %r13 + + pop %r15 + pop %r14 + pop %r13 + pop %r12 + pop %rbx + pop %rbp + ret + .endfn diagnose_syscall,globl diff --git a/libc/calls/getcontext.S b/libc/calls/getcontext.S index b14199b58..0bef8e9a8 100644 --- a/libc/calls/getcontext.S +++ b/libc/calls/getcontext.S @@ -20,10 +20,26 @@ // Gets machine state. // -// @note please use longerjmp() and setlongerjmp() for fibers -// @note currently only gets general registers // @see setcontext() getcontext: + pushf + pop 176(%rdi) + movaps %xmm0,480(%rdi) + movaps %xmm1,496(%rdi) + movaps %xmm2,512(%rdi) + movaps %xmm3,528(%rdi) + movaps %xmm4,544(%rdi) + movaps %xmm5,560(%rdi) + movaps %xmm6,576(%rdi) + movaps %xmm7,592(%rdi) + movaps %xmm8,608(%rdi) + movaps %xmm9,624(%rdi) + movaps %xmm10,640(%rdi) + movaps %xmm11,656(%rdi) + movaps %xmm12,672(%rdi) + movaps %xmm13,688(%rdi) + movaps %xmm14,704(%rdi) + movaps %xmm15,720(%rdi) mov %r8,40(%rdi) mov %r9,48(%rdi) mov %r10,56(%rdi) @@ -39,6 +55,8 @@ getcontext: mov %rdx,136(%rdi) mov %rax,144(%rdi) mov %rcx,152(%rdi) + lea 320(%rdi),%rax + mov %rax,224(%rdi) lea 8(%rsp),%rax mov %rax,160(%rdi) mov (%rsp),%rax @@ -50,6 +68,22 @@ getcontext: .end //////////////////////////////////////////////////////////////////////////////// noasan noubsan noinstrument int getcontext(ucontext_t *uc) { + asm volatile("movaps\t%%xmm0,%0" : /* no outputs */ : "m"(uc->__fpustate.xmm[0])); + asm volatile("movaps\t%%xmm1,%0" : /* no outputs */ : "m"(uc->__fpustate.xmm[1])); + asm volatile("movaps\t%%xmm2,%0" : /* no outputs */ : "m"(uc->__fpustate.xmm[2])); + asm volatile("movaps\t%%xmm3,%0" : /* no outputs */ : "m"(uc->__fpustate.xmm[3])); + asm volatile("movaps\t%%xmm4,%0" : /* no outputs */ : "m"(uc->__fpustate.xmm[4])); + asm volatile("movaps\t%%xmm5,%0" : /* no outputs */ : "m"(uc->__fpustate.xmm[5])); + asm volatile("movaps\t%%xmm6,%0" : /* no outputs */ : "m"(uc->__fpustate.xmm[6])); + asm volatile("movaps\t%%xmm7,%0" : /* no outputs */ : "m"(uc->__fpustate.xmm[7])); + asm volatile("movaps\t%%xmm8,%0" : /* no outputs */ : "m"(uc->__fpustate.xmm[8])); + asm volatile("movaps\t%%xmm9,%0" : /* no outputs */ : "m"(uc->__fpustate.xmm[9])); + asm volatile("movaps\t%%xmm10,%0" : /* no outputs */ : "m"(uc->__fpustate.xmm[10])); + asm volatile("movaps\t%%xmm11,%0" : /* no outputs */ : "m"(uc->__fpustate.xmm[11])); + asm volatile("movaps\t%%xmm12,%0" : /* no outputs */ : "m"(uc->__fpustate.xmm[12])); + asm volatile("movaps\t%%xmm13,%0" : /* no outputs */ : "m"(uc->__fpustate.xmm[13])); + asm volatile("movaps\t%%xmm14,%0" : /* no outputs */ : "m"(uc->__fpustate.xmm[14])); + asm volatile("movaps\t%%xmm15,%0" : /* no outputs */ : "m"(uc->__fpustate.xmm[15])); asm volatile("mov\t%%r8,%0" : "=m"(uc->uc_mcontext.r8)); asm volatile("mov\t%%r9,%0" : "=m"(uc->uc_mcontext.r9)); asm volatile("mov\t%%r10,%0" : "=m"(uc->uc_mcontext.r10)); @@ -65,6 +99,7 @@ noasan noubsan noinstrument int getcontext(ucontext_t *uc) { asm volatile("mov\t%%rdx,%0" : "=m"(uc->uc_mcontext.rdx)); asm volatile("mov\t%%rax,%0" : "=m"(uc->uc_mcontext.rax)); asm volatile("mov\t%%rcx,%0" : "=m"(uc->uc_mcontext.rcx)); + uc->uc_mcontext.fpregs = &uc->__fpustate; asm volatile("lea\t8(%%rsp),%%rax\n\t" "mov\t%%rax,%0" : "=m"(uc->uc_mcontext.rsp) diff --git a/libc/calls/setcontext.S b/libc/calls/setcontext.S index 958e8b1eb..a48919d18 100644 --- a/libc/calls/setcontext.S +++ b/libc/calls/setcontext.S @@ -20,10 +20,29 @@ // Sets machine state. // -// @note please use longerjmp() and setlongerjmp() for fibers -// @note currently only sets general registers // @see getcontext() setcontext: + mov 224(%rdi),%rax + test %rax,%rax + je 1f + movaps 160(%rax),%xmm0 + movaps 176(%rax),%xmm1 + movaps 192(%rax),%xmm2 + movaps 208(%rax),%xmm3 + movaps 224(%rax),%xmm4 + movaps 240(%rax),%xmm5 + movaps 256(%rax),%xmm6 + movaps 272(%rax),%xmm7 + movaps 288(%rax),%xmm8 + movaps 304(%rax),%xmm9 + movaps 320(%rax),%xmm10 + movaps 336(%rax),%xmm11 + movaps 352(%rax),%xmm12 + movaps 368(%rax),%xmm13 + movaps 384(%rax),%xmm14 + movaps 400(%rax),%xmm15 +1: push 176(%rdi) + popf mov 40(%rdi),%r8 mov 48(%rdi),%r9 mov 56(%rdi),%r10 @@ -48,6 +67,24 @@ setcontext: .end //////////////////////////////////////////////////////////////////////////////// noasan noubsan int setcontext(const ucontext_t *uc) { + if (uc->uc_mcontext.fpregs) { + asm volatile("movaps\t%0,%%xmm0" : /* no outputs */ : "m"(uc->uc_mcontext.fpregs->xmm[0])); + asm volatile("movaps\t%0,%%xmm1" : /* no outputs */ : "m"(uc->uc_mcontext.fpregs->xmm[1])); + asm volatile("movaps\t%0,%%xmm2" : /* no outputs */ : "m"(uc->uc_mcontext.fpregs->xmm[2])); + asm volatile("movaps\t%0,%%xmm3" : /* no outputs */ : "m"(uc->uc_mcontext.fpregs->xmm[3])); + asm volatile("movaps\t%0,%%xmm4" : /* no outputs */ : "m"(uc->uc_mcontext.fpregs->xmm[4])); + asm volatile("movaps\t%0,%%xmm5" : /* no outputs */ : "m"(uc->uc_mcontext.fpregs->xmm[5])); + asm volatile("movaps\t%0,%%xmm6" : /* no outputs */ : "m"(uc->uc_mcontext.fpregs->xmm[6])); + asm volatile("movaps\t%0,%%xmm7" : /* no outputs */ : "m"(uc->uc_mcontext.fpregs->xmm[7])); + asm volatile("movaps\t%0,%%xmm8" : /* no outputs */ : "m"(uc->uc_mcontext.fpregs->xmm[8])); + asm volatile("movaps\t%0,%%xmm9" : /* no outputs */ : "m"(uc->uc_mcontext.fpregs->xmm[9])); + asm volatile("movaps\t%0,%%xmm10" : /* no outputs */ : "m"(uc->uc_mcontext.fpregs->xmm[10])); + asm volatile("movaps\t%0,%%xmm11" : /* no outputs */ : "m"(uc->uc_mcontext.fpregs->xmm[11])); + asm volatile("movaps\t%0,%%xmm12" : /* no outputs */ : "m"(uc->uc_mcontext.fpregs->xmm[12])); + asm volatile("movaps\t%0,%%xmm13" : /* no outputs */ : "m"(uc->uc_mcontext.fpregs->xmm[13])); + asm volatile("movaps\t%0,%%xmm14" : /* no outputs */ : "m"(uc->uc_mcontext.fpregs->xmm[14])); + asm volatile("movaps\t%0,%%xmm15" : /* no outputs */ : "m"(uc->uc_mcontext.fpregs->xmm[15])); + } asm volatile("mov\t%0,%%r8" : /* no outputs */ : "m"(uc->uc_mcontext.r8)); asm volatile("mov\t%0,%%r9" : /* no outputs */ : "m"(uc->uc_mcontext.r9)); asm volatile("mov\t%0,%%r10" : /* no outputs */ : "m"(uc->uc_mcontext.r10)); diff --git a/libc/calls/struct/ucontext-netbsd.internal.h b/libc/calls/struct/ucontext-netbsd.internal.h index 8d297c174..ec8326312 100644 --- a/libc/calls/struct/ucontext-netbsd.internal.h +++ b/libc/calls/struct/ucontext-netbsd.internal.h @@ -1,9 +1,9 @@ #ifndef COSMOPOLITAN_LIBC_CALLS_STRUCT_UCONTEXT_NETBSD_INTERNAL_H_ #define COSMOPOLITAN_LIBC_CALLS_STRUCT_UCONTEXT_NETBSD_INTERNAL_H_ +#include "libc/calls/ucontext.h" #if !(__ASSEMBLER__ + __LINKER__ + 0) COSMOPOLITAN_C_START_ // clang-format off -#include "libc/calls/ucontext.h" #define __UCONTEXT_SIZE 784 #define _UC_SIGMASK 0x01 diff --git a/libc/calls/ucontext.h b/libc/calls/ucontext.h index 493275043..05be425fd 100644 --- a/libc/calls/ucontext.h +++ b/libc/calls/ucontext.h @@ -65,7 +65,7 @@ struct MachineContext { }; gregset_t gregs; }; - struct FpuState *fpregs; + struct FpuState *fpregs; /* zero when no fpu context */ uint64_t __pad1[8]; }; @@ -77,6 +77,7 @@ struct ucontext { stack_t uc_stack; mcontext_t uc_mcontext; /* use this */ sigset_t uc_sigmask; + uint64_t __pad; struct FpuState __fpustate; /* for cosmo on non-linux */ }; diff --git a/libc/integral/normalize.inc b/libc/integral/normalize.inc index 5618d7eb0..513778444 100644 --- a/libc/integral/normalize.inc +++ b/libc/integral/normalize.inc @@ -68,11 +68,11 @@ /* TODO(jart): Remove this in favor of GetStackSize() */ #if defined(COSMO) && defined(MODE_DBG) -#define STACKSIZE 0x20000 /* 128kb stack */ +#define STACKSIZE 131072 /* 128kb stack */ #elif defined(COSMO) -#define STACKSIZE 0x10000 /* 64kb stack */ +#define STACKSIZE 65536 /* 64kb stack */ #else -#define STACKSIZE 0x800000 /* 8mb stack */ +#define STACKSIZE 8388608 /* 8mb stack */ #endif #define BIGPAGESIZE 0x200000 diff --git a/libc/intrin/_have_fsgsbase.c b/libc/intrin/_have_fsgsbase.c new file mode 100644 index 000000000..5a8779849 --- /dev/null +++ b/libc/intrin/_have_fsgsbase.c @@ -0,0 +1,63 @@ +/*-*- mode:c;indent-tabs-mode:nil;c-basic-offset:2;tab-width:8;coding:utf-8 -*-│ +│vi: set net ft=c ts=2 sts=2 sw=2 fenc=utf-8 :vi│ +╞══════════════════════════════════════════════════════════════════════════════╡ +│ Copyright 2022 Justine Alexandra Roberts Tunney │ +│ │ +│ Permission to use, copy, modify, and/or distribute this software for │ +│ any purpose with or without fee is hereby granted, provided that the │ +│ above copyright notice and this permission notice appear in all copies. │ +│ │ +│ THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL │ +│ WARRANTIES WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED │ +│ WARRANTIES OF MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE │ +│ AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL │ +│ DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR │ +│ PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER │ +│ TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR │ +│ PERFORMANCE OF THIS SOFTWARE. │ +╚─────────────────────────────────────────────────────────────────────────────*/ +#include "libc/dce.h" +#include "libc/errno.h" +#include "libc/intrin/fsgsbase.h" +#include "libc/nexgen32e/x86feature.h" + +/** + * Returns true if FSGSBASE ISA can be used. + * + * If this function returns true (Linux 5.9+ or FreeBSD) then you should + * be able to read/write to the %gs / %fs x86 segment registers in about + * one or two clock cycles which gives you a free addition operation for + * all assembly ops that reference memory. + * + * The FSGSBASE ISA was introduced by Intel with Ivybridge (c. 2012) but + * the Linux Kernel didn't authorize us to use it until 2020, once Intel + * had to start backdooring customer kernels so that they could have it. + * AMD introduced support for the FSGSBASE ISA in Excavator, aka bdver4. + * + * @return boolean indicating if feature can be used + * @see _rdfsbase() + * @see _rdgsbase() + * @see _wrfsbase() + * @see _wrgsbase() + */ +privileged int _have_fsgsbase(void) { + // Linux 5.9 (c. 2020) introduced close_range() and fsgsbase support. + // it's cheaper to test for close_range() than handle an op crashing. + // Windows lets us use these instructions but they don't really work. + int ax; + if (X86_HAVE(FSGSBASE)) { + if (IsLinux()) { + asm volatile("syscall" + : "=a"(ax) + : "0"(436 /* close_range */), "D"(-1), "S"(-2), "d"(0) + : "rcx", "r11", "memory"); + return ax == -22; // EINVAL + } else if (IsFreebsd()) { + return 1; + } else { + return 0; + } + } else { + return 0; + } +} diff --git a/libc/intrin/_rdfsbase.c b/libc/intrin/_rdfsbase.c new file mode 100644 index 000000000..839dbf807 --- /dev/null +++ b/libc/intrin/_rdfsbase.c @@ -0,0 +1,28 @@ +/*-*- mode:c;indent-tabs-mode:nil;c-basic-offset:2;tab-width:8;coding:utf-8 -*-│ +│vi: set net ft=c ts=2 sts=2 sw=2 fenc=utf-8 :vi│ +╞══════════════════════════════════════════════════════════════════════════════╡ +│ Copyright 2022 Justine Alexandra Roberts Tunney │ +│ │ +│ Permission to use, copy, modify, and/or distribute this software for │ +│ any purpose with or without fee is hereby granted, provided that the │ +│ above copyright notice and this permission notice appear in all copies. │ +│ │ +│ THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL │ +│ WARRANTIES WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED │ +│ WARRANTIES OF MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE │ +│ AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL │ +│ DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR │ +│ PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER │ +│ TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR │ +│ PERFORMANCE OF THIS SOFTWARE. │ +╚─────────────────────────────────────────────────────────────────────────────*/ +#include "libc/intrin/fsgsbase.h" + +/** + * Reads `%fs` base address. + * + * @see _have_fsgsbase() + */ +void *(_rdfsbase)(void) { + return _rdfsbase(); +} diff --git a/libc/intrin/_rdgsbase.c b/libc/intrin/_rdgsbase.c new file mode 100644 index 000000000..1d6d30d73 --- /dev/null +++ b/libc/intrin/_rdgsbase.c @@ -0,0 +1,28 @@ +/*-*- mode:c;indent-tabs-mode:nil;c-basic-offset:2;tab-width:8;coding:utf-8 -*-│ +│vi: set net ft=c ts=2 sts=2 sw=2 fenc=utf-8 :vi│ +╞══════════════════════════════════════════════════════════════════════════════╡ +│ Copyright 2022 Justine Alexandra Roberts Tunney │ +│ │ +│ Permission to use, copy, modify, and/or distribute this software for │ +│ any purpose with or without fee is hereby granted, provided that the │ +│ above copyright notice and this permission notice appear in all copies. │ +│ │ +│ THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL │ +│ WARRANTIES WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED │ +│ WARRANTIES OF MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE │ +│ AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL │ +│ DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR │ +│ PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER │ +│ TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR │ +│ PERFORMANCE OF THIS SOFTWARE. │ +╚─────────────────────────────────────────────────────────────────────────────*/ +#include "libc/intrin/fsgsbase.h" + +/** + * Reads `%gs` base address. + * + * @see _have_fsgsbase() + */ +void *(_rdgsbase)(void) { + return _rdgsbase(); +} diff --git a/libc/intrin/_wrfsbase.c b/libc/intrin/_wrfsbase.c new file mode 100644 index 000000000..e8dab5528 --- /dev/null +++ b/libc/intrin/_wrfsbase.c @@ -0,0 +1,28 @@ +/*-*- mode:c;indent-tabs-mode:nil;c-basic-offset:2;tab-width:8;coding:utf-8 -*-│ +│vi: set net ft=c ts=2 sts=2 sw=2 fenc=utf-8 :vi│ +╞══════════════════════════════════════════════════════════════════════════════╡ +│ Copyright 2022 Justine Alexandra Roberts Tunney │ +│ │ +│ Permission to use, copy, modify, and/or distribute this software for │ +│ any purpose with or without fee is hereby granted, provided that the │ +│ above copyright notice and this permission notice appear in all copies. │ +│ │ +│ THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL │ +│ WARRANTIES WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED │ +│ WARRANTIES OF MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE │ +│ AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL │ +│ DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR │ +│ PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER │ +│ TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR │ +│ PERFORMANCE OF THIS SOFTWARE. │ +╚─────────────────────────────────────────────────────────────────────────────*/ +#include "libc/intrin/fsgsbase.h" + +/** + * Changes `%fs` base address. + * + * @see _have_fsgsbase() + */ +void *(_wrfsbase)(void *p) { + return _wrfsbase(p); +} diff --git a/libc/intrin/_wrgsbase.c b/libc/intrin/_wrgsbase.c new file mode 100644 index 000000000..5e63bd1e0 --- /dev/null +++ b/libc/intrin/_wrgsbase.c @@ -0,0 +1,28 @@ +/*-*- mode:c;indent-tabs-mode:nil;c-basic-offset:2;tab-width:8;coding:utf-8 -*-│ +│vi: set net ft=c ts=2 sts=2 sw=2 fenc=utf-8 :vi│ +╞══════════════════════════════════════════════════════════════════════════════╡ +│ Copyright 2022 Justine Alexandra Roberts Tunney │ +│ │ +│ Permission to use, copy, modify, and/or distribute this software for │ +│ any purpose with or without fee is hereby granted, provided that the │ +│ above copyright notice and this permission notice appear in all copies. │ +│ │ +│ THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL │ +│ WARRANTIES WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED │ +│ WARRANTIES OF MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE │ +│ AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL │ +│ DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR │ +│ PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER │ +│ TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR │ +│ PERFORMANCE OF THIS SOFTWARE. │ +╚─────────────────────────────────────────────────────────────────────────────*/ +#include "libc/intrin/fsgsbase.h" + +/** + * Changes `%gs` base address. + * + * @see _have_fsgsbase() + */ +void *(_wrgsbase)(void *p) { + return _wrgsbase(p); +} diff --git a/libc/intrin/describearchprctlcode.c b/libc/intrin/describearchprctlcode.c new file mode 100644 index 000000000..6ebcd76a9 --- /dev/null +++ b/libc/intrin/describearchprctlcode.c @@ -0,0 +1,30 @@ +/*-*- mode:c;indent-tabs-mode:nil;c-basic-offset:2;tab-width:8;coding:utf-8 -*-│ +│vi: set net ft=c ts=2 sts=2 sw=2 fenc=utf-8 :vi│ +╞══════════════════════════════════════════════════════════════════════════════╡ +│ Copyright 2022 Justine Alexandra Roberts Tunney │ +│ │ +│ Permission to use, copy, modify, and/or distribute this software for │ +│ any purpose with or without fee is hereby granted, provided that the │ +│ above copyright notice and this permission notice appear in all copies. │ +│ │ +│ THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL │ +│ WARRANTIES WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED │ +│ WARRANTIES OF MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE │ +│ AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL │ +│ DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR │ +│ PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER │ +│ TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR │ +│ PERFORMANCE OF THIS SOFTWARE. │ +╚─────────────────────────────────────────────────────────────────────────────*/ +#include "libc/calls/calls.h" +#include "libc/fmt/itoa.h" +#include "libc/intrin/describeflags.internal.h" + +const char *(DescribeArchPrctlCode)(char buf[12], int x) { + if (x == ARCH_SET_FS) return "ARCH_SET_FS"; + if (x == ARCH_GET_FS) return "ARCH_GET_FS"; + if (x == ARCH_SET_GS) return "ARCH_SET_GS"; + if (x == ARCH_GET_GS) return "ARCH_GET_GS"; + FormatInt32(buf, x); + return buf; +} diff --git a/libc/intrin/describeflags.internal.h b/libc/intrin/describeflags.internal.h index 34a526948..0bc065de1 100644 --- a/libc/intrin/describeflags.internal.h +++ b/libc/intrin/describeflags.internal.h @@ -12,6 +12,7 @@ struct thatispacked DescribeFlags { const char *DescribeFlags(char *, size_t, struct DescribeFlags *, size_t, const char *, unsigned); +const char *DescribeArchPrctlCode(char[12], int); const char *DescribeCapability(char[20], int); const char *DescribeClockName(char[32], int); const char *DescribeDirfd(char[12], int); @@ -54,6 +55,7 @@ const char *DescribeSocketProtocol(char[12], int); const char *DescribeSocketType(char[64], int); const char *DescribeWhence(char[12], int); +#define DescribeArchPrctlCode(x) DescribeArchPrctlCode(alloca(12), x) #define DescribeCapability(x) DescribeCapability(alloca(20), x) #define DescribeClockName(x) DescribeClockName(alloca(32), x) #define DescribeDirfd(x) DescribeDirfd(alloca(12), x) diff --git a/libc/intrin/fsgsbase.h b/libc/intrin/fsgsbase.h new file mode 100644 index 000000000..aeaf46ca8 --- /dev/null +++ b/libc/intrin/fsgsbase.h @@ -0,0 +1,47 @@ +#ifndef COSMOPOLITAN_LIBC_RUNTIME_FSGSBASE_H_ +#define COSMOPOLITAN_LIBC_RUNTIME_FSGSBASE_H_ +#if !(__ASSEMBLER__ + __LINKER__ + 0) +COSMOPOLITAN_C_START_ + +void *_rdfsbase(void); +void *_rdgsbase(void); +void *_wrfsbase(void *); +void *_wrgsbase(void *); +int _have_fsgsbase(void); + +#if defined(__GNUC__) && !defined(__STRICT_ANSI__) +#define _rdfsbase() \ + ({ \ + void *_p; \ + asm("rdfsbase\t%0" : "=r"(_p)); \ + _p; \ + }) +#define _rdgsbase() \ + ({ \ + void *_p; \ + asm("rdgsbase\t%0" : "=r"(_p)); \ + _p; \ + }) +#define _wrfsbase(p) \ + ({ \ + void *_p = p; \ + asm volatile("wrfsbase\t%0" \ + : /* no outputs */ \ + : "r"(_p) \ + : "memory"); \ + _p; \ + }) +#define _wrgsbase(p) \ + ({ \ + void *_p = p; \ + asm volatile("wrgsbase\t%0" \ + : /* no outputs */ \ + : "r"(_p) \ + : "memory"); \ + _p; \ + }) +#endif /* GNUC && !ANSI */ + +COSMOPOLITAN_C_END_ +#endif /* !(__ASSEMBLER__ + __LINKER__ + 0) */ +#endif /* COSMOPOLITAN_LIBC_RUNTIME_FSGSBASE_H_ */ diff --git a/libc/thread/pthread_attr_getstack.c b/libc/intrin/pthread_attr_getstack.c similarity index 100% rename from libc/thread/pthread_attr_getstack.c rename to libc/intrin/pthread_attr_getstack.c diff --git a/libc/intrin/pthread_attr_init.c b/libc/intrin/pthread_attr_init.c index ee00ed36f..b5471795e 100644 --- a/libc/intrin/pthread_attr_init.c +++ b/libc/intrin/pthread_attr_init.c @@ -30,5 +30,8 @@ int pthread_attr_init(pthread_attr_t *attr) { .stacksize = GetStackSize(), .guardsize = PAGESIZE, }; + if (attr->stacksize >= 1048576) { + attr->guardsize = FRAMESIZE; + } return 0; } diff --git a/libc/intrin/pthread_attr_setguardsize.c b/libc/intrin/pthread_attr_setguardsize.c index 4391aafc6..27f5077e5 100644 --- a/libc/intrin/pthread_attr_setguardsize.c +++ b/libc/intrin/pthread_attr_setguardsize.c @@ -16,21 +16,15 @@ │ TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR │ │ PERFORMANCE OF THIS SOFTWARE. │ ╚─────────────────────────────────────────────────────────────────────────────*/ -#include "libc/errno.h" #include "libc/intrin/pthread.h" -#include "libc/macros.internal.h" /** * Sets size of unmapped pages at bottom of stack. * - * Cosmopolitan Libc stack guards always default to 4096 bytes. Setting - * `guardsize` to zero will disable automatic creation of guard pages. - * Your `guardsize` will be rounded up to `PAGESIZE`. - * * @param guardsize contains guard size in bytes * @return 0 on success, or errno on error */ int pthread_attr_setguardsize(pthread_attr_t *attr, size_t guardsize) { - attr->guardsize = ROUNDUP(guardsize, PAGESIZE); + attr->guardsize = guardsize; return 0; } diff --git a/libc/thread/pthread_attr_setstack.c b/libc/intrin/pthread_attr_setstack.c similarity index 80% rename from libc/thread/pthread_attr_setstack.c rename to libc/intrin/pthread_attr_setstack.c index dedd4793b..6994b97d8 100644 --- a/libc/thread/pthread_attr_setstack.c +++ b/libc/intrin/pthread_attr_setstack.c @@ -16,19 +16,10 @@ │ TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR │ │ PERFORMANCE OF THIS SOFTWARE. │ ╚─────────────────────────────────────────────────────────────────────────────*/ -#include "libc/calls/calls.h" -#include "libc/calls/syscall-sysv.internal.h" #include "libc/dce.h" #include "libc/errno.h" #include "libc/intrin/asan.internal.h" #include "libc/intrin/pthread.h" -#include "libc/macros.internal.h" -#include "libc/runtime/stack.h" -#include "libc/sysv/consts/map.h" -#include "libc/sysv/consts/prot.h" - -#define MAP_ANON_OPENBSD 0x1000 -#define MAP_STACK_OPENBSD 0x4000 /** * Configures custom allocated stack for thread, e.g. @@ -82,30 +73,6 @@ int pthread_attr_setstack(pthread_attr_t *attr, void *stackaddr, (IsAsan() && !__asan_is_valid(stackaddr, stacksize))) { return EINVAL; } - if (IsOpenbsd()) { - // OpenBSD: Only permits RSP to occupy memory that's been explicitly - // defined as stack memory. We need to squeeze the provided interval - // in order to successfully call mmap(), which will return EINVAL if - // these calculations should overflow. - size_t n; - int e, rc; - uintptr_t x, y; - n = stacksize; - x = (uintptr_t)stackaddr; - y = ROUNDUP(x, PAGESIZE); - n -= y - x; - n = ROUNDDOWN(n, PAGESIZE); - stackaddr = (void *)y; - stacksize = n; - e = errno; - if (__sys_mmap(stackaddr, stacksize, PROT_READ | PROT_WRITE, - MAP_PRIVATE | MAP_ANON_OPENBSD | MAP_STACK_OPENBSD, -1, 0, - 0) == MAP_FAILED) { - rc = errno; - errno = e; - return rc; - } - } attr->stackaddr = stackaddr; attr->stacksize = stacksize; return 0; diff --git a/libc/intrin/pthread_attr_setstacksize.c b/libc/intrin/pthread_attr_setstacksize.c index 0c89c831c..893f68714 100644 --- a/libc/intrin/pthread_attr_setstacksize.c +++ b/libc/intrin/pthread_attr_setstacksize.c @@ -20,34 +20,14 @@ #include "libc/intrin/pthread.h" /** - * Sets size of thread stack. + * Defines minimum stack size for thread. * - * Your stack must have at least `PTHREAD_STACK_MIN` bytes, which - * Cosmpolitan Libc defines as `GetStackSize()`. It's a link-time - * constant used by Actually Portable Executable that's 128 kb by - * default. See libc/runtime/stack.h for docs on your stack limit - * since the APE ELF phdrs are the one true source of truth here. - * - * Cosmpolitan Libc runtime magic (e.g. ftrace) and memory safety - * (e.g. kprintf) assumes that stack sizes are two-powers and are - * aligned to that two-power. Conformance isn't required since we - * say caveat emptor to those who don't maintain these invariants - * - * Unlike pthread_attr_setstack() this function should be used if - * you want the Cosmopolitan Libc runtime to allocate a stack for - * you. Since the runtime uses mmap(MAP_STACK) to do that, you'll - * need to choose a multiple of FRAMESIZE, due to Windows. - * - * If this function isn't called it'll default to GetStackSize(). - * - * @param x contains stack size in bytes + * @param stacksize contains stack size in bytes * @return 0 on success, or errno on error - * @raise EINVAL if `x` is less than `PTHREAD_STACK_MIN` + * @raise EINVAL if `stacksize` is less than `PTHREAD_STACK_MIN` */ -int pthread_attr_setstacksize(pthread_attr_t *a, size_t x) { - if (x < PTHREAD_STACK_MIN) return EINVAL; - if (x & (FRAMESIZE - 1)) return EINVAL; - if (x < FRAMESIZE) return EINVAL; - a->stacksize = x; +int pthread_attr_setstacksize(pthread_attr_t *a, size_t stacksize) { + if (stacksize < PTHREAD_STACK_MIN) return EINVAL; + a->stacksize = stacksize; return 0; } diff --git a/libc/intrin/sys_gettid.greg.c b/libc/intrin/sys_gettid.greg.c index eb57853f1..82cab9d92 100644 --- a/libc/intrin/sys_gettid.greg.c +++ b/libc/intrin/sys_gettid.greg.c @@ -38,24 +38,24 @@ privileged int sys_gettid(void) { asm("syscall" // xnu/osfmk/kern/ipc_tt.c : "=a"(tid) // assume success : "0"(0x1000000 | 27) // Mach thread_self_trap() - : "rcx", "r11", "memory", "cc"); + : "rcx", "r8", "r9", "r10", "r11", "memory", "cc"); } else if (IsOpenbsd()) { asm("syscall" : "=a"(tid) // man says always succeeds : "0"(299) // getthrid() - : "rcx", "r11", "memory", "cc"); + : "rcx", "r8", "r9", "r10", "r11", "memory", "cc"); } else if (IsNetbsd()) { asm("syscall" : "=a"(tid) // man says always succeeds : "0"(311) // _lwp_self() - : "rcx", "rdx", "r11", "memory", "cc"); + : "rcx", "rdx", "r8", "r9", "r10", "r11", "memory", "cc"); } else if (IsFreebsd()) { asm("syscall" : "=a"(tid), // only fails w/ EFAULT, which isn't possible "=m"(wut) // must be 64-bit : "0"(432), // thr_self() "D"(&wut) // but not actually 64-bit - : "rcx", "r11", "memory", "cc"); + : "rcx", "r8", "r9", "r10", "r11", "memory", "cc"); tid = wut; } else { tid = __pid; diff --git a/libc/runtime/arch_prctl.c b/libc/runtime/arch_prctl.c index 6ee17d98b..abad88f08 100644 --- a/libc/runtime/arch_prctl.c +++ b/libc/runtime/arch_prctl.c @@ -17,31 +17,20 @@ │ PERFORMANCE OF THIS SOFTWARE. │ ╚─────────────────────────────────────────────────────────────────────────────*/ #include "libc/calls/calls.h" +#include "libc/calls/strace.internal.h" #include "libc/calls/syscall-sysv.internal.h" #include "libc/dce.h" #include "libc/errno.h" +#include "libc/intrin/asan.internal.h" #include "libc/intrin/asmflag.h" #include "libc/intrin/bits.h" +#include "libc/intrin/describeflags.internal.h" #include "libc/nexgen32e/msr.h" #include "libc/nexgen32e/x86feature.h" #include "libc/runtime/pc.internal.h" #include "libc/sysv/consts/sig.h" #include "libc/sysv/errfuns.h" -/** - * @fileoverview Memory segmentation system calls. - * - * This whole file basically does: - * - * mov foo,%fs - * mov foo,%gs - * mov %fs,foo - * mov %gs,foo - * - * Which is nontrivial due to the limitless authoritarianism of - * operating systems. - */ - #define rdmsr(msr) \ ({ \ uint32_t lo, hi; \ @@ -58,24 +47,7 @@ "d"((uint32_t)(val_ >> 32))); \ } while (0) -static inline int arch_prctl_fsgsbase(int code, int64_t addr) { - switch (code) { - case ARCH_SET_GS: - asm volatile("wrgsbase\t%0" : /* no outputs */ : "r"(addr)); - return 0; - case ARCH_SET_FS: - asm volatile("wrfsbase\t%0" : /* no outputs */ : "r"(addr)); - return 0; - case ARCH_GET_GS: - asm volatile("rdgsbase\t%0" : "=r"(*(int64_t *)addr)); - return 0; - case ARCH_GET_FS: - asm volatile("rdfsbase\t%0" : "=r"(*(int64_t *)addr)); - return 0; - default: - return einval(); - } -} +int sys_enable_tls(); static int arch_prctl_msr(int code, int64_t addr) { switch (code) { @@ -99,29 +71,52 @@ static int arch_prctl_msr(int code, int64_t addr) { static int arch_prctl_freebsd(int code, int64_t addr) { switch (code) { case ARCH_GET_FS: + // sysarch(AMD64_GET_FSBASE) return sys_arch_prctl(128, addr); case ARCH_SET_FS: - return sys_arch_prctl(129, addr); + // sysarch(AMD64_SET_FSBASE) + return sys_arch_prctl(129, (intptr_t)&addr); case ARCH_GET_GS: + // sysarch(AMD64_GET_GSBASE) return sys_arch_prctl(130, addr); case ARCH_SET_GS: - return sys_arch_prctl(131, addr); + // sysarch(AMD64_SET_GSBASE) + return sys_arch_prctl(131, (intptr_t)&addr); default: return einval(); } } -static privileged dontinline int arch_prctl_xnu(int code, int64_t addr) { - int ax; - bool failed; +static int arch_prctl_netbsd(int code, int64_t addr) { + switch (code) { + case ARCH_GET_FS: + // sysarch(X86_GET_FSBASE) + return sys_arch_prctl(15, addr); + case ARCH_SET_FS: + // we use _lwp_setprivate() instead of sysarch(X86_SET_FSBASE) + // because the latter has a bug where signal handlers cause it + // to be clobbered. please note, this doesn't apply to %gs :-) + return sys_enable_tls(addr); + case ARCH_GET_GS: + // sysarch(X86_GET_GSBASE) + return sys_arch_prctl(14, addr); + case ARCH_SET_GS: + // sysarch(X86_SET_GSBASE) + return sys_arch_prctl(16, (intptr_t)&addr); + default: + return einval(); + } +} + +static int arch_prctl_xnu(int code, int64_t addr) { + int e; switch (code) { case ARCH_SET_GS: - asm volatile(CFLAG_ASM("syscall") - : CFLAG_CONSTRAINT(failed), "=a"(ax) - : "1"(0x3000003), "D"(addr - 0x30) - : "rcx", "r11", "memory", "cc"); - if (failed) errno = ax, ax = -1; - return ax; + // thread_fast_set_cthread_self has a weird ABI + e = errno; + sys_enable_tls(addr); + errno = e; + return 0; case ARCH_GET_FS: case ARCH_SET_FS: case ARCH_GET_GS: @@ -139,7 +134,8 @@ static privileged dontinline int arch_prctl_openbsd(int code, int64_t addr) { asm volatile(CFLAG_ASM("syscall") : CFLAG_CONSTRAINT(failed), "=a"(rax) : "1"(0x014a /* __get_tcb */) - : "rcx", "r11", "cc", "memory"); + : "rcx", "rdx", "rdi", "rsi", "r8", "r9", "r10", "r11", "cc", + "memory"); if (failed) { errno = rax; return -1; @@ -150,7 +146,8 @@ static privileged dontinline int arch_prctl_openbsd(int code, int64_t addr) { asm volatile(CFLAG_ASM("syscall") : CFLAG_CONSTRAINT(failed), "=a"(rax) : "1"(0x0149 /* __set_tcb */), "D"(addr) - : "rcx", "r11", "cc", "memory"); + : "rcx", "rdx", "rsi", "r8", "r9", "r10", "r11", "cc", + "memory"); if (failed) { errno = rax; rax = -1; @@ -164,27 +161,80 @@ static privileged dontinline int arch_prctl_openbsd(int code, int64_t addr) { } } -static char g_fsgs_once; - /** - * Don't bother. + * Changes x86 segment registers. + * + * Support for segment registers is spotty across platforms. See the + * table of tested combinations below. + * + * This wrapper has the same weird ABI as the Linux Kernel. The type + * Cosmopolitan type signature of the prototype for this function is + * variadic, so no value safety checking will be performed w/o asan. + * + * Cosmopolitan Libc initializes your process by default, to use the + * segment register %fs, for thread-local storage. To safely disable + * this TLS you need to either set `__tls_enabled` to 0, or you must + * follow the same memory layout assumptions as your C library. When + * TLS is disabled you can't use threads unless you call clone() and + * that's not really a good idea since `errno` won't be thread-safe. + * + * Please note if you're only concerned about running on Linux, then + * consider using Cosmopolitan's fsgsbase macros which don't need to + * issue system calls to change %fs and %gs. See _have_fsgsbase() to + * learn more. + * + * @param code may be + * - `ARCH_SET_FS` works on Linux, FreeBSD, NetBSD, OpenBSD, Metal + * - `ARCH_GET_FS` works on Linux, FreeBSD, NetBSD, OpenBSD, Metal + * - `ARCH_SET_GS` works on Linux, FreeBSD, NetBSD, XNU, Metal + * - `ARCH_GET_GS` works on Linux, FreeBSD, NetBSD, Metal + * @param addr is treated as `intptr_t` when setting a register, and + * is an output parameter (i.e. `intptr_t *`) when reading one + * @raise ENOSYS if operating system didn't support changing `code` + * @raise EINVAL if `code` wasn't valid + * @raise EFAULT if `ARCH_SET_FS` or `ARCH_SET_GS` was used and memory + * pointed to by `addr` was invalid + * @see _have_fsgsbase() */ int arch_prctl(int code, int64_t addr) { - void *fn = arch_prctl_fsgsbase; - switch (__hostos) { - case METAL: - return arch_prctl_msr(code, addr); - case FREEBSD: - // TODO(jart): this should use sysarch() - return arch_prctl_freebsd(code, addr); - case OPENBSD: - return arch_prctl_openbsd(code, addr); - case LINUX: - return sys_arch_prctl(code, addr); - case XNU: - /* probably won't work */ - return arch_prctl_xnu(code, addr); - default: - return enosys(); + int rc; + if (IsAsan() && // + (code == ARCH_GET_FS || // + code == ARCH_GET_GS) && // + !__asan_is_valid((int64_t *)addr, 8)) { + rc = efault(); + } else { + switch (__hostos) { + case METAL: + rc = arch_prctl_msr(code, addr); + break; + case FREEBSD: + rc = arch_prctl_freebsd(code, addr); + break; + case NETBSD: + rc = arch_prctl_netbsd(code, addr); + break; + case OPENBSD: + rc = arch_prctl_openbsd(code, addr); + break; + case LINUX: + rc = sys_arch_prctl(code, addr); + break; + case XNU: + rc = arch_prctl_xnu(code, addr); + break; + default: + rc = enosys(); + break; + } } +#ifdef SYSDEBUG + if (!rc && (code == ARCH_GET_FS || code == ARCH_GET_GS)) { + STRACE("arch_prctl(%s, [%p]) → %d% m", DescribeArchPrctlCode(code), + *(int64_t *)addr, rc); + } else { + STRACE("arch_prctl(%s, %p) → %d% m", DescribeArchPrctlCode(code), addr, rc); + } +#endif + return rc; } diff --git a/libc/runtime/clone.c b/libc/runtime/clone.c index cce8f608d..ebddb8d08 100644 --- a/libc/runtime/clone.c +++ b/libc/runtime/clone.c @@ -172,7 +172,7 @@ XnuThreadMain(void *pthread, // rdi asm volatile("syscall" : "=a"(ax) : "0"(__NR_thread_fast_set_cthread_self), "D"(wt->tls - 0x30) - : "rcx", "r11", "memory", "cc"); + : "rcx", "rdx", "r8", "r9", "r10", "r11", "memory", "cc"); } func(arg, tid); @@ -388,7 +388,7 @@ static int CloneNetbsd(int (*func)(void *, int), char *stk, size_t stksz, asm volatile(CFLAG_ASM("syscall") : CFLAG_CONSTRAINT(failed), "=a"(ax) : "1"(__NR_getcontext_netbsd), "D"(&netbsd_clone_template) - : "rcx", "rdx", "r11", "memory"); + : "rcx", "rdx", "r8", "r9", "r10", "r11", "memory"); if (failed) { broken = ax; } @@ -443,7 +443,7 @@ static int CloneNetbsd(int (*func)(void *, int), char *stk, size_t stksz, asm volatile(CFLAG_ASM("syscall") : CFLAG_CONSTRAINT(failed), "=a"(ax), "=d"(dx) : "1"(__NR__lwp_create), "D"(ctx), "S"(LWP_DETACHED), "2"(tid) - : "rcx", "r11", "memory"); + : "rcx", "r8", "r9", "r10", "r11", "memory"); if (!failed) { return *tid; } else { diff --git a/libc/runtime/enable_tls.c b/libc/runtime/enable_tls.c index 090f3cd95..b6916d131 100644 --- a/libc/runtime/enable_tls.c +++ b/libc/runtime/enable_tls.c @@ -51,6 +51,8 @@ #define _TLDZ ((intptr_t)_tdata_size) #define _TIBZ sizeof(struct cthread_descriptor_t) +int sys_enable_tls(); + typedef char xmm_t __attribute__((__vector_size__(16), __aligned__(1))); __msabi extern typeof(TlsAlloc) *const __imp_TlsAlloc; @@ -136,34 +138,21 @@ privileged void __enable_tls(void) { assert(0 <= __tls_index && __tls_index < 64); asm("mov\t%1,%%gs:%0" : "=m"(*((long *)0x1480 + __tls_index)) : "r"(tib)); } else if (IsFreebsd()) { - asm volatile("syscall" - : "=a"(ax) - : "0"(__NR_sysarch), "D"(AMD64_SET_FSBASE), "S"(tib) - : "rcx", "r11", "memory", "cc"); + sys_enable_tls(AMD64_SET_FSBASE, tib); + } else if (IsLinux()) { + sys_enable_tls(ARCH_SET_FS, tib); } else if (IsNetbsd()) { // netbsd has sysarch(X86_SET_FSBASE) but we can't use that because - // signal handlers will cause it to be reset due to net setting the + // signal handlers will cause it to be reset due to not setting the // _mc_tlsbase field in struct mcontext_netbsd. - asm volatile("syscall" - : "=a"(ax), "=d"(dx) - : "0"(__NR__lwp_setprivate), "D"(tib) - : "rcx", "r11", "memory", "cc"); - } else if (IsXnu()) { - asm volatile("syscall" - : "=a"(ax) - : "0"(__NR_thread_fast_set_cthread_self), - "D"((intptr_t)tib - 0x30) - : "rcx", "r11", "memory", "cc"); + sys_enable_tls(tib); } else if (IsOpenbsd()) { - asm volatile("syscall" - : "=a"(ax) - : "0"(__NR___set_tcb), "D"(tib) - : "rcx", "r11", "memory", "cc"); - } else if (IsLinux()) { - asm volatile("syscall" - : "=a"(ax) - : "0"(__NR_linux_arch_prctl), "D"(ARCH_SET_FS), "S"(tib) - : "rcx", "r11", "memory"); + sys_enable_tls(tib); + } else if (IsXnu()) { + // thread_fast_set_cthread_self has a weird ABI + int e = errno; + sys_enable_tls((intptr_t)tib - 0x30); + errno = e; } else { uint64_t val = (uint64_t)tib; asm volatile("wrmsr" diff --git a/libc/runtime/ftracer.c b/libc/runtime/ftracer.c index 951bc9a51..44d7a611d 100644 --- a/libc/runtime/ftracer.c +++ b/libc/runtime/ftracer.c @@ -17,6 +17,7 @@ │ PERFORMANCE OF THIS SOFTWARE. │ ╚─────────────────────────────────────────────────────────────────────────────*/ #include "libc/calls/calls.h" +#include "libc/errno.h" #include "libc/fmt/itoa.h" #include "libc/intrin/cmpxchg.h" #include "libc/intrin/kprintf.h" @@ -24,6 +25,7 @@ #include "libc/macros.internal.h" #include "libc/nexgen32e/gettls.h" #include "libc/nexgen32e/stackframe.h" +#include "libc/nexgen32e/threaded.h" #include "libc/runtime/internal.h" #include "libc/runtime/stack.h" #include "libc/runtime/symbols.internal.h" @@ -42,6 +44,7 @@ void ftrace_hook(void); static int g_stackdigs; +static struct Ftrace g_ftrace; static privileged inline int GetNestingLevelImpl(struct StackFrame *frame) { int nesting = -2; @@ -52,7 +55,7 @@ static privileged inline int GetNestingLevelImpl(struct StackFrame *frame) { return MAX(0, nesting); } -static privileged inline int GetNestingLevel(struct FtraceTls *ft, +static privileged inline int GetNestingLevel(struct Ftrace *ft, struct StackFrame *sf) { int nesting; nesting = GetNestingLevelImpl(sf); @@ -70,9 +73,13 @@ static privileged inline int GetNestingLevel(struct FtraceTls *ft, */ privileged void ftracer(void) { long stackuse; - struct FtraceTls *ft; + struct Ftrace *ft; struct StackFrame *sf; - ft = (struct FtraceTls *)(__get_tls_privileged() + 0x08); + if (__tls_enabled) { + ft = (struct Ftrace *)(__get_tls_privileged() + 0x08); + } else { + ft = &g_ftrace; + } if (_cmpxchg(&ft->once, false, true)) { ft->lastaddr = -1; ft->skew = GetNestingLevelImpl(__builtin_frame_address(0)); diff --git a/libc/runtime/morph.greg.c b/libc/runtime/morph.greg.c index 1fc32b61a..24aa69724 100644 --- a/libc/runtime/morph.greg.c +++ b/libc/runtime/morph.greg.c @@ -18,12 +18,12 @@ ╚─────────────────────────────────────────────────────────────────────────────*/ #define ShouldUseMsabiAttribute() 1 #include "libc/assert.h" -#include "libc/intrin/asmflag.h" #include "libc/calls/internal.h" #include "libc/calls/strace.internal.h" #include "libc/calls/struct/sigset.h" #include "libc/dce.h" #include "libc/errno.h" +#include "libc/intrin/asmflag.h" #include "libc/intrin/kprintf.h" #include "libc/nt/enum/pageflags.h" #include "libc/nt/memory.h" @@ -50,7 +50,7 @@ static privileged void __morph_mprotect(void *addr, size_t size, int prot, "syscall") : CFLAG_CONSTRAINT(cf), "=a"(ax), "=d"(dx) : "1"(__NR_mprotect), "D"(addr), "S"(size), "2"(prot) - : "rcx", "r11", "memory"); + : "rcx", "r8", "r9", "r10", "r11", "memory"); #ifndef NDEBUG if (cf) ax = -ax; if (ax == -EPERM) { @@ -58,6 +58,7 @@ static privileged void __morph_mprotect(void *addr, size_t size, int prot, _Exit(26); } #endif + if (ax) notpossible; } else { __imp_VirtualProtect(addr, size, ntprot, &op); } @@ -81,15 +82,15 @@ privileged void __morph_begin(void) { : "=a"(ax), "=d"(dx) : "0"(__NR_sigprocmask), "D"(SIG_BLOCK), "S"(&ss), "1"(&oldss) - : "rcx", "r10", "r11", "memory", "cc"); - assert(!ax); + : "rcx", "r8", "r9", "r10", "r11", "memory", "cc"); + if (ax) notpossible; } else { asm volatile(CFLAG_ASM("syscall") : CFLAG_CONSTRAINT(cf), "=a"(ax), "=d"(dx) : "1"(__NR_sigprocmask), "D"(SIG_BLOCK), "S"(-1u) - : "rcx", "r11", "memory"); + : "rcx", "r8", "r9", "r10", "r11", "memory"); oldss.__bits[0] = ax & 0xffffffff; - assert(!cf); + if (cf) notpossible; } } __morph_mprotect(_base, __privileged_addr - _base, PROT_READ | PROT_WRITE, @@ -112,15 +113,15 @@ privileged void __morph_end(void) { : "=a"(ax), "=d"(dx) : "0"(__NR_sigprocmask), "D"(SIG_SETMASK), "S"(&oldss), "1"(0) - : "rcx", "r10", "r11", "memory", "cc"); - assert(!ax); + : "rcx", "r8", "r9", "r10", "r11", "memory", "cc"); + if (ax) notpossible; } else { asm volatile(CFLAG_ASM("syscall") : CFLAG_CONSTRAINT(cf), "=a"(ax), "=d"(dx) : "1"(__NR_sigprocmask), "D"(SIG_SETMASK), "S"(oldss.__bits[0]) - : "rcx", "r11", "memory"); - assert(!cf); + : "rcx", "r8", "r9", "r10", "r11", "memory"); + if (cf) notpossible; } } STRACE("__morph_end()"); diff --git a/libc/sysv/calls/sys_arch_prctl.s b/libc/sysv/calls/sys_arch_prctl.s index 555afcaa7..5ea06eae8 100644 --- a/libc/sysv/calls/sys_arch_prctl.s +++ b/libc/sysv/calls/sys_arch_prctl.s @@ -1,2 +1,2 @@ .include "o/libc/sysv/macros.internal.inc" -.scall sys_arch_prctl,0xfff0a50a5ffff09e,globl,hidden +.scall sys_arch_prctl,0x0a50a50a5ffff09e,globl,hidden diff --git a/libc/sysv/calls/sys_enable_tls.s b/libc/sysv/calls/sys_enable_tls.s new file mode 100644 index 000000000..9abf27093 --- /dev/null +++ b/libc/sysv/calls/sys_enable_tls.s @@ -0,0 +1,2 @@ +.include "o/libc/sysv/macros.internal.inc" +.scall sys_enable_tls,0x13d1490a5300309e,globl,hidden diff --git a/libc/sysv/consts/uc.h b/libc/sysv/consts/uc.h new file mode 100644 index 000000000..8f0b6fde8 --- /dev/null +++ b/libc/sysv/consts/uc.h @@ -0,0 +1,12 @@ +#ifndef COSMOPOLITAN_LIBC_SYSV_CONSTS_UC_H_ +#define COSMOPOLITAN_LIBC_SYSV_CONSTS_UC_H_ +#if !(__ASSEMBLER__ + __LINKER__ + 0) +COSMOPOLITAN_C_START_ + +#define UC_FP_XSTATE 1 +#define UC_SIGCONTEXT_SS 2 +#define UC_STRICT_RESTORE_SS 4 + +COSMOPOLITAN_C_END_ +#endif /* !(__ASSEMBLER__ + __LINKER__ + 0) */ +#endif /* COSMOPOLITAN_LIBC_SYSV_CONSTS_UC_H_ */ diff --git a/libc/sysv/syscalls.sh b/libc/sysv/syscalls.sh index ef71e3a9a..cd8cccee1 100755 --- a/libc/sysv/syscalls.sh +++ b/libc/sysv/syscalls.sh @@ -212,7 +212,8 @@ scall modify_ldt 0xfffffffffffff09a globl scall sys_pivot_root 0xfffffffffffff09b globl hidden scall _sysctl 0xfffffffffffff09c globl #scall prctl 0xfffffffffffff09d globl # wrapped manually -scall sys_arch_prctl 0xfff0a50a5ffff09e globl hidden # sysarch() on bsd +scall sys_arch_prctl 0x0a50a50a5ffff09e globl hidden # sysarch() on bsd +scall sys_enable_tls 0x13d1490a5300309e globl hidden # arch_prctl on linux, sysarch on freebsd, _lwp_setprivate on netbsd, __set_tcb on openbsd, _lwp_setprivate on netbsd, thread_fast_set_cthread_self on xnu scall adjtimex 0xfffffffffffff09f globl scall swapon 0xffffff05520550a7 globl scall swapoff 0xffffff1a8ffff0a8 globl diff --git a/libc/testlib/testrunner.c b/libc/testlib/testrunner.c index 818577c54..bd67be048 100644 --- a/libc/testlib/testrunner.c +++ b/libc/testlib/testrunner.c @@ -28,7 +28,6 @@ #include "libc/fmt/fmt.h" #include "libc/fmt/itoa.h" #include "libc/intrin/atomic.h" -#include "libc/intrin/kprintf.h" #include "libc/intrin/pthread.h" #include "libc/intrin/weaken.h" #include "libc/log/check.h" diff --git a/libc/thread/posixthread.internal.h b/libc/thread/posixthread.internal.h index 15b42b0c8..69e5abcf3 100644 --- a/libc/thread/posixthread.internal.h +++ b/libc/thread/posixthread.internal.h @@ -2,7 +2,6 @@ #define COSMOPOLITAN_LIBC_THREAD_POSIXTHREAD_INTERNAL_H_ #include "libc/intrin/pthread.h" #include "libc/runtime/runtime.h" -#include "libc/thread/spawn.h" #if !(__ASSEMBLER__ + __LINKER__ + 0) COSMOPOLITAN_C_START_ @@ -54,13 +53,16 @@ enum PosixThreadStatus { }; struct PosixThread { - struct spawn spawn; void *(*start_routine)(void *); void *arg; // start_routine's parameter void *rc; // start_routine's return value + bool ownstack; + int tid; + int *ctid; + char *tls; + char *tib; _Atomic(enum PosixThreadStatus) status; jmp_buf exiter; - size_t stacksize; pthread_attr_t attr; }; diff --git a/libc/thread/pthread_create.c b/libc/thread/pthread_create.c index 25a6cd15a..4def12abb 100644 --- a/libc/thread/pthread_create.c +++ b/libc/thread/pthread_create.c @@ -18,17 +18,22 @@ ╚─────────────────────────────────────────────────────────────────────────────*/ #include "libc/assert.h" #include "libc/calls/calls.h" +#include "libc/calls/syscall-sysv.internal.h" +#include "libc/dce.h" #include "libc/errno.h" #include "libc/intrin/asan.internal.h" #include "libc/intrin/atomic.h" +#include "libc/intrin/bits.h" #include "libc/intrin/pthread.h" #include "libc/intrin/wait0.internal.h" #include "libc/intrin/weaken.h" +#include "libc/macros.internal.h" #include "libc/mem/mem.h" #include "libc/nexgen32e/gc.internal.h" #include "libc/nexgen32e/gettls.h" #include "libc/nexgen32e/threaded.h" #include "libc/runtime/runtime.h" +#include "libc/runtime/stack.h" #include "libc/sysv/consts/clone.h" #include "libc/sysv/consts/map.h" #include "libc/sysv/consts/prot.h" @@ -37,14 +42,19 @@ #include "libc/thread/spawn.h" #include "libc/thread/thread.h" +#define MAP_ANON_OPENBSD 0x1000 +#define MAP_STACK_OPENBSD 0x4000 + void _pthread_wait(struct PosixThread *pt) { - _wait0(pt->spawn.ctid); + _wait0(pt->ctid); } void _pthread_free(struct PosixThread *pt) { - free(pt->spawn.tls); - if (pt->stacksize) { - munmap(&pt->spawn.stk, pt->stacksize); + free(pt->tls); + if (pt->ownstack && // + pt->attr.stackaddr && // + pt->attr.stackaddr != MAP_FAILED) { + munmap(&pt->attr.stackaddr, pt->attr.stacksize); } free(pt); } @@ -71,6 +81,36 @@ static int PosixThread(void *arg, int tid) { return 0; } +static int FixupCustomStackOnOpenbsd(pthread_attr_t *attr) { + // OpenBSD: Only permits RSP to occupy memory that's been explicitly + // defined as stack memory. We need to squeeze the provided interval + // in order to successfully call mmap(), which will return EINVAL if + // these calculations should overflow. + size_t n; + int e, rc; + uintptr_t x, y; + n = attr->stacksize; + x = (uintptr_t)attr->stackaddr; + y = ROUNDUP(x, PAGESIZE); + n -= y - x; + n = ROUNDDOWN(n, PAGESIZE); + e = errno; + if (__sys_mmap((void *)y, n, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANON_OPENBSD | MAP_STACK_OPENBSD, -1, 0, + 0) != MAP_FAILED) { + attr->stackaddr = (void *)y; + attr->stacksize = n; + return 0; + } else { + rc = errno; + errno = e; + if (rc == EOVERFLOW) { + rc = EINVAL; + } + return rc; + } +} + /** * Creates thread, e.g. * @@ -124,16 +164,9 @@ int pthread_create(pthread_t *thread, const pthread_attr_t *attr, void *(*start_routine)(void *), void *arg) { int rc, e = errno; struct PosixThread *pt; - pthread_attr_t default_attr; TlsIsRequired(); _pthread_zombies_decimate(); - // default attributes - if (!attr) { - pthread_attr_init(&default_attr); - attr = &default_attr; - } - // create posix thread object if (!(pt = calloc(1, sizeof(struct PosixThread)))) { errno = e; @@ -143,26 +176,48 @@ int pthread_create(pthread_t *thread, const pthread_attr_t *attr, pt->arg = arg; // create thread local storage memory - if (!(pt->spawn.tls = _mktls(&pt->spawn.tib))) { + if (!(pt->tls = _mktls(&pt->tib))) { free(pt); errno = e; return EAGAIN; } // child thread id is also a condition variable - pt->spawn.ctid = (int *)(pt->spawn.tib + 0x38); + pt->ctid = (int *)(pt->tib + 0x38); - // create stack - if (attr && attr->stackaddr) { - // caller is responsible for creating stacks - pt->spawn.stk = attr->stackaddr; + // setup attributes + if (attr) { + pt->attr = *attr; + attr = 0; } else { - // cosmo posix threads is managing the stack - pt->spawn.stk = mmap(0, attr->stacksize, PROT_READ | PROT_WRITE, - MAP_STACK | MAP_ANONYMOUS, -1, 0); - if (pt->spawn.stk != MAP_FAILED) { - pt->stacksize = attr->stacksize; - } else { + pthread_attr_init(&pt->attr); + } + + // setup stack + if (pt->attr.stackaddr) { + // caller supplied their own stack + // assume they know what they're doing as much as possible + if (IsOpenbsd()) { + if ((rc = FixupCustomStackOnOpenbsd(&pt->attr))) { + _pthread_free(pt); + return rc; + } + } + } else { + // cosmo is managing the stack + // 1. in mono repo optimize for tiniest stack possible + // 2. in public world optimize to *work* regardless of memory + pt->ownstack = true; + pt->attr.stacksize = MAX(pt->attr.stacksize, GetStackSize()); + pt->attr.stacksize = roundup2pow(pt->attr.stacksize); + pt->attr.guardsize = ROUNDUP(pt->attr.guardsize, PAGESIZE); + if (pt->attr.guardsize + PAGESIZE >= pt->attr.stacksize) { + _pthread_free(pt); + return EINVAL; + } + pt->attr.stackaddr = mmap(0, pt->attr.stacksize, PROT_READ | PROT_WRITE, + MAP_STACK | MAP_ANONYMOUS, -1, 0); + if (pt->attr.stackaddr == MAP_FAILED) { rc = errno; _pthread_free(pt); errno = e; @@ -173,41 +228,31 @@ int pthread_create(pthread_t *thread, const pthread_attr_t *attr, } } // mmap(MAP_STACK) creates a 4096 guard by default - if (attr->guardsize != PAGESIZE) { - // user requested special guard size - if (attr->guardsize) { - rc = mprotect(pt->spawn.stk, attr->guardsize, PROT_NONE); + if (pt->attr.guardsize != PAGESIZE) { + if (pt->attr.guardsize) { + // user requested special guard size + rc = mprotect(pt->attr.stackaddr, pt->attr.guardsize, PROT_NONE); } else { - rc = mprotect(pt->spawn.stk, PAGESIZE, PROT_READ | PROT_WRITE); + // user explicitly disabled guard page + rc = mprotect(pt->attr.stackaddr, PAGESIZE, PROT_READ | PROT_WRITE); } if (rc) { notpossible; } } if (IsAsan()) { - if (attr->guardsize) { - __asan_poison(pt->spawn.stk, attr->guardsize, kAsanStackOverflow); + if (pt->attr.guardsize) { + __asan_poison(pt->attr.stackaddr, pt->attr.guardsize, + kAsanStackOverflow); } - __asan_poison( - pt->spawn.stk + attr->stacksize - 16 /* openbsd:stackbound */, 16, - kAsanStackOverflow); + __asan_poison((char *)pt->attr.stackaddr + pt->attr.stacksize - + 16 /* openbsd:stackbound */, + 16, kAsanStackOverflow); } } - // we only need to save this to support pthread_getattr_np() - pt->attr = *attr; - // if attr->stackaddr == 0, - // the stack is managed by cosmo, - // pt->spawn.stk is from a successful mmap, - // and so pt->attr.stackaddr = pt->spawn.stk - pt->attr.stackaddr = pt->spawn.stk; - // if attr->stackaddr != 0, - // then stack is not managed by cosmo - // pt->attr.stackaddr = pt->spawn.stk = attr->stackaddr - // so the above line is a no-op. - // set initial status - switch (attr->detachstate) { + switch (pt->attr.detachstate) { case PTHREAD_CREATE_JOINABLE: pt->status = kPosixThreadJoinable; break; @@ -221,20 +266,16 @@ int pthread_create(pthread_t *thread, const pthread_attr_t *attr, } // launch PosixThread(pt) in new thread - if (clone(PosixThread, pt->spawn.stk, - attr->stacksize - 16 /* openbsd:stackbound */, + if (clone(PosixThread, pt->attr.stackaddr, + pt->attr.stacksize - 16 /* openbsd:stackbound */, CLONE_VM | CLONE_THREAD | CLONE_FS | CLONE_FILES | CLONE_SIGHAND | CLONE_SETTLS | CLONE_PARENT_SETTID | CLONE_CHILD_SETTID | CLONE_CHILD_CLEARTID, - pt, &pt->spawn.ptid, pt->spawn.tib, pt->spawn.ctid) == -1) { + pt, &pt->tid, pt->tib, pt->ctid) == -1) { rc = errno; _pthread_free(pt); errno = e; - if (rc == EINVAL) { - return EINVAL; - } else { - return EAGAIN; - } + return rc; } if (thread) { diff --git a/libc/thread/pthread_equal.c b/libc/thread/pthread_equal.c index 1fb1ae0ce..c7e3afcce 100644 --- a/libc/thread/pthread_equal.c +++ b/libc/thread/pthread_equal.c @@ -27,5 +27,5 @@ int pthread_equal(pthread_t t1, pthread_t t2) { struct PosixThread *a = (struct PosixThread *)t1; struct PosixThread *b = (struct PosixThread *)t2; - return a->spawn.ptid == b->spawn.ptid; + return a->tid == b->tid; } diff --git a/libc/thread/pthread_getname_np.c b/libc/thread/pthread_getname_np.c index d2c796c57..a306fcd79 100644 --- a/libc/thread/pthread_getname_np.c +++ b/libc/thread/pthread_getname_np.c @@ -50,7 +50,7 @@ int pthread_getname_np(pthread_t thread, char *name, size_t size) { if (!size) return 0; bzero(name, size); - tid = ((struct PosixThread *)thread)->spawn.ptid; + tid = ((struct PosixThread *)thread)->tid; if (IsLinux()) { // TASK_COMM_LEN is 16 on Linux so we're just being paranoid. @@ -99,7 +99,7 @@ int pthread_getname_np(pthread_t thread, char *name, size_t size) { : CFLAG_CONSTRAINT(cf), "=a"(ax), "=d"(dx) : "1"(324 /* _lwp_getname */), "D"(tid), "S"(name), "d"(size - 1) - : "rcx", "r11", "memory"); + : "rcx", "r8", "r9", "r10", "r11", "memory"); if (!cf) { // if size + our nul + kernel's nul is the buffer size, then we // can't say with absolute confidence truncation didn't happen. diff --git a/libc/thread/pthread_getunique_np.c b/libc/thread/pthread_getunique_np.c index a7e046508..8e8b58637 100644 --- a/libc/thread/pthread_getunique_np.c +++ b/libc/thread/pthread_getunique_np.c @@ -24,5 +24,5 @@ */ int64_t pthread_getunique_np(pthread_t thread) { struct PosixThread *pt = (struct PosixThread *)thread; - return pt->spawn.ptid; + return pt->tid; } diff --git a/libc/thread/pthread_setname_np.c b/libc/thread/pthread_setname_np.c index 1ea57df50..0663b05db 100644 --- a/libc/thread/pthread_setname_np.c +++ b/libc/thread/pthread_setname_np.c @@ -56,7 +56,7 @@ int pthread_setname_np(pthread_t thread, const char *name) { char path[128], *p; int e, fd, rc, tid, len; - tid = ((struct PosixThread *)thread)->spawn.ptid; + tid = ((struct PosixThread *)thread)->tid; len = strlen(name); if (IsLinux()) { @@ -100,7 +100,7 @@ int pthread_setname_np(pthread_t thread, const char *name) { asm volatile(CFLAG_ASM("syscall") : CFLAG_CONSTRAINT(cf), "=a"(ax), "=d"(dx) : "1"(323 /* thr_set_name */), "D"(tid), "S"(name) - : "rcx", "r11", "memory"); + : "rcx", "r8", "r9", "r10", "r11", "memory"); return !cf ? 0 : ax; } else if (IsNetbsd()) { @@ -109,7 +109,7 @@ int pthread_setname_np(pthread_t thread, const char *name) { asm volatile(CFLAG_ASM("syscall") : CFLAG_CONSTRAINT(cf), "=a"(ax), "=d"(dx) : "1"(323 /* _lwp_setname */), "D"(tid), "S"(name) - : "rcx", "r11", "memory"); + : "rcx", "r8", "r9", "r10", "r11", "memory"); return !cf ? 0 : ax; } else { diff --git a/libc/thread/thread.h b/libc/thread/thread.h index 0bde1fa03..1d5b5cc0f 100644 --- a/libc/thread/thread.h +++ b/libc/thread/thread.h @@ -6,7 +6,7 @@ #if !(__ASSEMBLER__ + __LINKER__ + 0) COSMOPOLITAN_C_START_ -struct FtraceTls { /* 16 */ +struct Ftrace { /* 16 */ bool once; /* 0 */ bool noreentry; /* 1 */ int skew; /* 4 */ @@ -15,7 +15,7 @@ struct FtraceTls { /* 16 */ struct cthread_descriptor_t { struct cthread_descriptor_t *self; /* 0x00 */ - struct FtraceTls ftrace; /* 0x08 */ + struct Ftrace ftrace; /* 0x08 */ void *garbages; /* 0x18 */ locale_t locale; /* 0x20 */ pthread_t pthread; /* 0x28 */ diff --git a/test/libc/calls/diagnose_syscall_test.c b/test/libc/calls/diagnose_syscall_test.c new file mode 100644 index 000000000..ae7515a25 --- /dev/null +++ b/test/libc/calls/diagnose_syscall_test.c @@ -0,0 +1,130 @@ +/*-*- mode:c;indent-tabs-mode:nil;c-basic-offset:2;tab-width:8;coding:utf-8 -*-│ +│vi: set net ft=c ts=2 sts=2 sw=2 fenc=utf-8 :vi│ +╞══════════════════════════════════════════════════════════════════════════════╡ +│ Copyright 2022 Justine Alexandra Roberts Tunney │ +│ │ +│ Permission to use, copy, modify, and/or distribute this software for │ +│ any purpose with or without fee is hereby granted, provided that the │ +│ above copyright notice and this permission notice appear in all copies. │ +│ │ +│ THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL │ +│ WARRANTIES WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED │ +│ WARRANTIES OF MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE │ +│ AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL │ +│ DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR │ +│ PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER │ +│ TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR │ +│ PERFORMANCE OF THIS SOFTWARE. │ +╚─────────────────────────────────────────────────────────────────────────────*/ +#include "libc/calls/calls.h" +#include "libc/calls/ucontext.h" +#include "libc/dce.h" +#include "libc/intrin/kprintf.h" +#include "libc/runtime/gc.h" +#include "libc/runtime/runtime.h" +#include "libc/stdio/append.internal.h" +#include "libc/str/str.h" +#include "libc/sysv/consts/nr.h" +#include "libc/testlib/testlib.h" + +#define Z 0x5555555555555555 + +#define FLAGS_cf 0 +#define FLAGS_pf 2 +#define FLAGS_sf 7 +#define FLAGS_of 11 + +intptr_t diagnose_syscall(intptr_t nr, // + intptr_t arg1, // + intptr_t arg2, // + intptr_t arg3, // + intptr_t arg4, // + intptr_t arg5, // + intptr_t arg6, // + intptr_t arg7, // + ucontext_t *before, // + ucontext_t *after); // + +#define GREG(FIELD) \ + do { \ + uint64_t f1 = x->uc_mcontext.FIELD; \ + uint64_t f2 = y->uc_mcontext.FIELD; \ + if (f1 != f2) { \ + if (b) appendw(&b, ' '); \ + appends(&b, #FIELD); \ + kprintf("%3s %016lx → %016lx\n", #FIELD, f1, f2); \ + } \ + } while (0) + +#define FLAG(FLAG) \ + if ((x->uc_mcontext.eflags & (1ul << FLAGS_##FLAG)) ^ \ + (y->uc_mcontext.eflags & (1ul << FLAGS_##FLAG))) { \ + if (b) appendw(&b, ' '); \ + appends(&b, #FLAG); \ + } + +char *DiffContexts(ucontext_t *x, ucontext_t *y) { + char *b = 0; + GREG(rax); + GREG(rdx); + GREG(rdi); + GREG(rsi); + GREG(rcx); + GREG(r8); + GREG(r9); + GREG(r10); + GREG(r11); + GREG(r12); + GREG(r13); + GREG(r14); + GREG(r15); + GREG(rbx); + GREG(rbp); + FLAG(cf); + FLAG(sf); + FLAG(of); + FLAG(pf); + return b; +} + +void SetUp(void) { + if (IsWindows()) { + exit(0); + } +} + +TEST(diagnose_syscall, getpid) { + ucontext_t x, y; + diagnose_syscall(__NR_getpid, Z, Z, Z, Z, Z, Z, Z, &x, &y); + if (IsFreebsd()) { + ASSERT_STREQ("rax rcx r8 r9 r10 r11", _gc(DiffContexts(&x, &y))); + } else if (IsNetbsd() || IsXnu()) { + // netbsd puts parent pid in edx + // xnu seems to just clobber it! + ASSERT_STREQ("rax rdx rcx r11", _gc(DiffContexts(&x, &y))); + } else { + ASSERT_STREQ("rax rcx r11", _gc(DiffContexts(&x, &y))); + } +} + +TEST(diagnose_syscall, testWriteSuccess) { + ucontext_t x, y; + diagnose_syscall(__NR_write, 2, Z, 0, Z, Z, Z, Z, &x, &y); + if (IsFreebsd()) { + ASSERT_STREQ("rax rcx r8 r9 r10 r11", _gc(DiffContexts(&x, &y))); + } else { + ASSERT_STREQ("rax rcx r11", _gc(DiffContexts(&x, &y))); + } +} + +TEST(diagnose_syscall, testWriteFailed) { + ucontext_t x, y; + diagnose_syscall(__NR_write, -1, Z, Z, Z, Z, Z, Z, &x, &y); + if (IsFreebsd()) { + ASSERT_STREQ("rax rcx r8 r9 r10 r11 cf", _gc(DiffContexts(&x, &y))); + } else if (IsBsd()) { + ASSERT_STREQ("rax rcx r11 cf", _gc(DiffContexts(&x, &y))); + } else { + ASSERT_STREQ("rax rcx r11", _gc(DiffContexts(&x, &y))); + } +} diff --git a/test/libc/calls/getcontext_test.c b/test/libc/calls/getcontext_test.c index 3a5faad0e..2308d62a9 100644 --- a/test/libc/calls/getcontext_test.c +++ b/test/libc/calls/getcontext_test.c @@ -19,6 +19,7 @@ #include "libc/calls/calls.h" #include "libc/calls/ucontext.h" #include "libc/runtime/runtime.h" +#include "libc/testlib/ezbench.h" #include "libc/testlib/testlib.h" int x; @@ -47,3 +48,17 @@ TEST(getcontext, test) { ASSERT_TRUE(ok1); ASSERT_TRUE(ok2); } + +void SetGetContext(void) { + static int a; + a = 0; + getcontext(&context); + if (!a) { + a = 1; + setcontext(&context); + } +} + +BENCH(getcontext, bench) { + EZBENCH2("get/setcontext", donothing, SetGetContext()); +} diff --git a/test/libc/intrin/pthread_mutex_lock2_test.c b/test/libc/intrin/pthread_mutex_lock2_test.c index f486e7cab..e8d8794f9 100644 --- a/test/libc/intrin/pthread_mutex_lock2_test.c +++ b/test/libc/intrin/pthread_mutex_lock2_test.c @@ -25,6 +25,7 @@ #include "libc/testlib/ezbench.h" #include "libc/testlib/testlib.h" #include "libc/thread/posixthread.internal.h" +#include "libc/thread/spawn.h" int THREADS = 16; int ITERATIONS = 100; diff --git a/test/libc/runtime/arch_prctl_test.c b/test/libc/runtime/arch_prctl_test.c index 007023079..6c1e9cda9 100644 --- a/test/libc/runtime/arch_prctl_test.c +++ b/test/libc/runtime/arch_prctl_test.c @@ -16,10 +16,18 @@ │ TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR │ │ PERFORMANCE OF THIS SOFTWARE. │ ╚─────────────────────────────────────────────────────────────────────────────*/ -#include "libc/intrin/segmentation.h" #include "libc/calls/calls.h" +#include "libc/calls/struct/sigaction.h" +#include "libc/calls/syscall-sysv.internal.h" #include "libc/dce.h" +#include "libc/errno.h" +#include "libc/intrin/fsgsbase.h" +#include "libc/intrin/kprintf.h" +#include "libc/intrin/segmentation.h" #include "libc/nexgen32e/threaded.h" +#include "libc/nt/version.h" +#include "libc/sysv/consts/sa.h" +#include "libc/sysv/consts/sig.h" #include "libc/testlib/testlib.h" void SetUpOnce(void) { @@ -27,19 +35,35 @@ void SetUpOnce(void) { ASSERT_SYS(0, 0, pledge("stdio rpath", 0)); } +void OnTrap(int sig, struct siginfo *si, void *vctx) { + struct ucontext *ctx = vctx; +} + +void TriggerSignal(void) { + struct sigaction old; + struct sigaction sig = {.sa_sigaction = OnTrap, .sa_flags = SA_SIGINFO}; + sched_yield(); + sigaction(SIGTRAP, &sig, &old); + asm("int3"); + sigaction(SIGTRAP, &old, 0); + sched_yield(); +} + TEST(arch_prctl, fs) { - if (IsLinux() || IsOpenbsd()) { + if (IsLinux() || IsFreebsd() || IsNetbsd() || IsOpenbsd()) { uint64_t n, x; x = 0xdeadbeef; arch_prctl(ARCH_SET_FS, &x); ASSERT_NE(-1, arch_prctl(ARCH_GET_FS, (intptr_t)&n)); ASSERT_EQ((intptr_t)&x, n); ASSERT_EQ(0xdeadbeef, fs((int64_t *)0)); + TriggerSignal(); + ASSERT_EQ(0xdeadbeef, fs((int64_t *)0)); } } TEST(arch_prctl, pointerRebasingFs) { - if (IsLinux() || IsOpenbsd()) { + if (IsLinux() || IsFreebsd() || IsOpenbsd() || IsNetbsd()) { unsigned long s[] = {0x0706050403020100, 0x0f0e0d0c0b0a0908}; ASSERT_EQ(0x0706050403020100, s[0]); ASSERT_EQ(0, arch_prctl(ARCH_SET_FS, 1)); @@ -53,26 +77,52 @@ TEST(arch_prctl, pointerRebasingFs) { } TEST(arch_prctl, gs) { - if (IsLinux()) { + if (IsLinux() || IsFreebsd() || IsNetbsd() || IsXnu()) { uint64_t n, x; x = 0xdeadbeef; arch_prctl(ARCH_SET_GS, &x); - ASSERT_NE(-1, arch_prctl(ARCH_GET_GS, (intptr_t)&n)); - ASSERT_EQ((intptr_t)&x, n); + if (!IsXnu()) { + ASSERT_NE(-1, arch_prctl(ARCH_GET_GS, (intptr_t)&n)); + ASSERT_EQ((intptr_t)&x, n); + } + ASSERT_EQ(0xdeadbeef, gs((int64_t *)0)); + TriggerSignal(); ASSERT_EQ(0xdeadbeef, gs((int64_t *)0)); } } TEST(arch_prctl, pointerRebasing) { - if (IsLinux()) { + if (IsLinux() || IsFreebsd() || IsNetbsd() || IsXnu()) { unsigned long s[] = {0x0706050403020100, 0x0f0e0d0c0b0a0908}; ASSERT_EQ(0x0706050403020100, s[0]); ASSERT_EQ(0, arch_prctl(ARCH_SET_GS, 1)); ASSERT_EQ(0x0807060504030201, gs(&s[0])); ASSERT_EQ(0, arch_prctl(ARCH_SET_GS, 2)); ASSERT_EQ(0x0908070605040302, gs(&s[0])); - intptr_t gs; - ASSERT_EQ(0, arch_prctl(ARCH_GET_GS, &gs)); - ASSERT_EQ(2, gs); + if (!IsXnu()) { + intptr_t gs; + ASSERT_EQ(0, arch_prctl(ARCH_GET_GS, &gs)); + ASSERT_EQ(2, gs); + } } } + +TEST(fsgsbase, fs) { + if (!_have_fsgsbase()) return; + int64_t mem = 0xdeadbeef; + _wrfsbase(&mem); + ASSERT_EQ(&mem, _rdfsbase()); + ASSERT_EQ(0xdeadbeef, fs((int64_t *)0)); + TriggerSignal(); + ASSERT_EQ(0xdeadbeef, fs((int64_t *)0)); +} + +TEST(fsgsbase, gs) { + if (!_have_fsgsbase()) return; + int64_t mem = 0xdeadbeef; + _wrgsbase(&mem); + ASSERT_EQ(&mem, _rdgsbase()); + ASSERT_EQ(0xdeadbeef, gs((int64_t *)0)); + TriggerSignal(); + ASSERT_EQ(0xdeadbeef, gs((int64_t *)0)); +} diff --git a/test/libc/thread/pthread_create_test.c b/test/libc/thread/pthread_create_test.c index c5bd1e04e..1e10d99ce 100644 --- a/test/libc/thread/pthread_create_test.c +++ b/test/libc/thread/pthread_create_test.c @@ -18,14 +18,18 @@ ╚─────────────────────────────────────────────────────────────────────────────*/ #include "libc/calls/calls.h" #include "libc/dce.h" +#include "libc/intrin/kprintf.h" #include "libc/intrin/pthread.h" +#include "libc/macros.internal.h" #include "libc/mem/mem.h" +#include "libc/runtime/runtime.h" #include "libc/runtime/stack.h" +#include "libc/sysv/consts/prot.h" +#include "libc/testlib/ezbench.h" #include "libc/testlib/subprocess.h" #include "libc/testlib/testlib.h" #include "libc/thread/thread.h" -#if 0 static void *Increment(void *arg) { ASSERT_EQ(gettid(), pthread_getthreadid_np()); return (void *)((uintptr_t)arg + 1); @@ -75,7 +79,7 @@ TEST(pthread_detach, testBigStack) { pthread_t id; pthread_attr_t attr; ASSERT_EQ(0, pthread_attr_init(&attr)); - ASSERT_EQ(0, pthread_attr_setstacksize(&attr, 2 * 1024 * 1024)); + ASSERT_EQ(0, pthread_attr_setstacksize(&attr, 2 * 1000 * 1000)); ASSERT_EQ(0, pthread_create(&id, &attr, CheckStack, 0)); ASSERT_EQ(0, pthread_attr_destroy(&attr)); ASSERT_EQ(0, pthread_join(id, 0)); @@ -101,7 +105,6 @@ TEST(pthread_detach, testCustomStack_withReallySmallSize) { ASSERT_EQ(0, pthread_join(id, 0)); free(stk); } -#endif TEST(pthread_exit, mainThreadWorks) { // _Exit1() can't set process exit code on XNU/NetBSD/OpenBSD. @@ -115,3 +118,31 @@ TEST(pthread_exit, mainThreadWorks) { EXITS(0); } } + +static void CreateJoin(void) { + pthread_t id; + ASSERT_EQ(0, pthread_create(&id, 0, Increment, 0)); + ASSERT_EQ(0, pthread_join(id, 0)); +} + +// this is de facto the same as create+join +static void CreateDetach(void) { + pthread_t id; + ASSERT_EQ(0, pthread_create(&id, 0, Increment, 0)); + ASSERT_EQ(0, pthread_detach(id)); +} + +// this is really fast +static void CreateDetached(void) { + pthread_attr_t attr; + ASSERT_EQ(0, pthread_attr_init(&attr)); + ASSERT_EQ(0, pthread_attr_setdetachstate(&attr, PTHREAD_CREATE_DETACHED)); + ASSERT_EQ(0, pthread_create(0, &attr, Increment, 0)); + ASSERT_EQ(0, pthread_attr_destroy(&attr)); +} + +BENCH(pthread_create, bench) { + EZBENCH2("CreateJoin", donothing, CreateJoin()); + EZBENCH2("CreateDetach", donothing, CreateDetach()); + EZBENCH2("CreateDetached", donothing, CreateDetached()); +}