From 0e2b1bfeed2654fa425a70f7d88e5d1306611a6d Mon Sep 17 00:00:00 2001 From: Justine Tunney Date: Thu, 8 Sep 2022 02:33:01 -0700 Subject: [PATCH] Make garbage collection thread safe - You can now use _gc(malloc()) in multithreaded programs - This change fixes a bug where fork() on NT disabled TLS - Fixed TLS code morphing on XNU/NT, for R8-R15 registers --- examples/compress.c | 1 + examples/decompress.c | 1 + libc/integral/c.inc | 8 +- libc/intrin/asan.c | 5 +- libc/intrin/tlsisrequired.c | 25 +++++ libc/log/backtrace2.c | 5 +- libc/log/backtrace3.c | 9 +- libc/log/log.mk | 1 + libc/log/printgarbage.c | 22 +++-- libc/mem/_gc_free.c | 24 +++++ libc/mem/defer.greg.c | 85 ----------------- libc/mem/gc.c | 92 ++++++++++++++++++- libc/nexgen32e/gc.S | 33 ++----- libc/nexgen32e/gc.internal.h | 5 +- libc/nexgen32e/gclongjmp.S | 20 ++-- libc/nexgen32e/threaded.h | 1 + libc/runtime/enable_tls.c | 4 +- libc/runtime/fork-nt.c | 3 +- libc/runtime/gc.h | 31 +++---- libc/thread/pthread_create.c | 5 + libc/thread/pthread_exit.c | 3 +- libc/thread/spawn.c | 25 ++++- libc/thread/thread.h | 3 +- .../ungarbage.c} | 34 +++---- test/libc/calls/open_test.c | 1 + test/libc/nexgen32e/gclongjmp_test.c | 41 +++++++++ test/libc/nexgen32e/test.mk | 1 + test/libc/sock/sendrecvmsg_test.c | 1 + test/libc/sock/test.mk | 1 + test/libc/stdio/fgetln_test.c | 1 + third_party/quickjs/call.c | 1 + third_party/smallz4/smallz4cat.c | 1 + third_party/tidy/config.c | 1 + tool/build/lib/elfwriter.c | 1 + tool/build/lib/psk.c | 2 + tool/hash/hash.mk | 1 + tool/viz/printimage.c | 1 + 37 files changed, 310 insertions(+), 189 deletions(-) create mode 100644 libc/intrin/tlsisrequired.c create mode 100644 libc/mem/_gc_free.c delete mode 100644 libc/mem/defer.greg.c rename libc/{log/printgarbagenumeric.c => thread/ungarbage.c} (67%) diff --git a/examples/compress.c b/examples/compress.c index 58ba9b83f..bd7b80461 100644 --- a/examples/compress.c +++ b/examples/compress.c @@ -8,6 +8,7 @@ ╚─────────────────────────────────────────────────────────────────*/ #endif #include "libc/assert.h" +#include "libc/calls/calls.h" #include "libc/errno.h" #include "libc/fmt/conv.h" #include "libc/log/check.h" diff --git a/examples/decompress.c b/examples/decompress.c index e85f52901..617da7732 100644 --- a/examples/decompress.c +++ b/examples/decompress.c @@ -8,6 +8,7 @@ ╚─────────────────────────────────────────────────────────────────*/ #endif #include "libc/assert.h" +#include "libc/calls/calls.h" #include "libc/errno.h" #include "libc/mem/mem.h" #include "libc/runtime/gc.internal.h" diff --git a/libc/integral/c.inc b/libc/integral/c.inc index 34b4098e9..c5aca2bc6 100644 --- a/libc/integral/c.inc +++ b/libc/integral/c.inc @@ -591,10 +591,10 @@ typedef struct { #endif #endif -#define notpossible \ - do { \ - asm("ud2"); \ - unreachable; \ +#define notpossible \ + do { \ + asm("ud2\nnop"); \ + unreachable; \ } while (0) #define donothing \ diff --git a/libc/intrin/asan.c b/libc/intrin/asan.c index ef29990a2..31064a01c 100644 --- a/libc/intrin/asan.c +++ b/libc/intrin/asan.c @@ -40,7 +40,9 @@ #include "libc/mem/mem.h" #include "libc/mem/reverse.internal.h" #include "libc/nexgen32e/gc.internal.h" +#include "libc/nexgen32e/gettls.h" #include "libc/nexgen32e/stackframe.h" +#include "libc/nexgen32e/threaded.h" #include "libc/nt/enum/version.h" #include "libc/nt/runtime.h" #include "libc/runtime/directmap.internal.h" @@ -56,6 +58,7 @@ #include "libc/sysv/consts/nr.h" #include "libc/sysv/consts/prot.h" #include "libc/sysv/errfuns.h" +#include "libc/thread/thread.h" #include "third_party/dlmalloc/dlmalloc.h" STATIC_YOINK("_init_asan"); @@ -936,7 +939,7 @@ static void __asan_trace(struct AsanTrace *bt, const struct StackFrame *bp) { size_t i, gi; intptr_t addr; struct Garbages *garbage; - garbage = weaken(__garbage); + garbage = __tls_enabled ? ((cthread_t)__get_tls())->garbages : 0; gi = garbage ? garbage->i : 0; for (f1 = -1, i = 0; bp && i < ARRAYLEN(bt->p); ++i, bp = bp->next) { if (f1 != (f2 = ((intptr_t)bp >> 16))) { diff --git a/libc/intrin/tlsisrequired.c b/libc/intrin/tlsisrequired.c new file mode 100644 index 000000000..a428d3144 --- /dev/null +++ b/libc/intrin/tlsisrequired.c @@ -0,0 +1,25 @@ +/*-*- mode:c;indent-tabs-mode:nil;c-basic-offset:2;tab-width:8;coding:utf-8 -*-│ +│vi: set net ft=c ts=2 sts=2 sw=2 fenc=utf-8 :vi│ +╞══════════════════════════════════════════════════════════════════════════════╡ +│ Copyright 2022 Justine Alexandra Roberts Tunney │ +│ │ +│ Permission to use, copy, modify, and/or distribute this software for │ +│ any purpose with or without fee is hereby granted, provided that the │ +│ above copyright notice and this permission notice appear in all copies. │ +│ │ +│ THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL │ +│ WARRANTIES WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED │ +│ WARRANTIES OF MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE │ +│ AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL │ +│ DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR │ +│ PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER │ +│ TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR │ +│ PERFORMANCE OF THIS SOFTWARE. │ +╚─────────────────────────────────────────────────────────────────────────────*/ +#include "libc/nexgen32e/threaded.h" + +void TlsIsRequired(void) { + if (!__tls_enabled) { + notpossible; + } +} diff --git a/libc/log/backtrace2.c b/libc/log/backtrace2.c index 52c4794ca..65b00e55e 100644 --- a/libc/log/backtrace2.c +++ b/libc/log/backtrace2.c @@ -37,6 +37,8 @@ #include "libc/mem/alg.h" #include "libc/mem/bisectcarleft.internal.h" #include "libc/nexgen32e/gc.internal.h" +#include "libc/nexgen32e/gettls.h" +#include "libc/nexgen32e/threaded.h" #include "libc/runtime/gc.internal.h" #include "libc/runtime/runtime.h" #include "libc/runtime/stack.h" @@ -47,6 +49,7 @@ #include "libc/sysv/consts/fileno.h" #include "libc/sysv/consts/o.h" #include "libc/sysv/consts/sig.h" +#include "libc/thread/thread.h" #include "libc/x/x.h" #define kBacktraceMaxFrames 128 @@ -106,7 +109,7 @@ static int PrintBacktraceUsingAddr2line(int fd, const struct StackFrame *bp) { argv[i++] = "-a"; /* filter out w/ shell script wrapper for old versions */ argv[i++] = "-pCife"; argv[i++] = debugbin; - garbage = weaken(__garbage); + garbage = __tls_enabled ? ((cthread_t)__get_tls())->garbages : 0; gi = garbage ? garbage->i : 0; for (frame = bp; frame && i < kBacktraceMaxFrames - 1; frame = frame->next) { addr = frame->addr; diff --git a/libc/log/backtrace3.c b/libc/log/backtrace3.c index 2d22fd022..eb07cc176 100644 --- a/libc/log/backtrace3.c +++ b/libc/log/backtrace3.c @@ -16,21 +16,24 @@ │ TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR │ │ PERFORMANCE OF THIS SOFTWARE. │ ╚─────────────────────────────────────────────────────────────────────────────*/ -#include "libc/mem/bisectcarleft.internal.h" #include "libc/assert.h" -#include "libc/intrin/weaken.h" #include "libc/calls/calls.h" #include "libc/fmt/fmt.h" #include "libc/fmt/itoa.h" #include "libc/intrin/kprintf.h" +#include "libc/intrin/weaken.h" #include "libc/log/backtrace.internal.h" #include "libc/macros.internal.h" +#include "libc/mem/bisectcarleft.internal.h" #include "libc/nexgen32e/gc.internal.h" +#include "libc/nexgen32e/gettls.h" #include "libc/nexgen32e/stackframe.h" +#include "libc/nexgen32e/threaded.h" #include "libc/runtime/memtrack.internal.h" #include "libc/runtime/runtime.h" #include "libc/runtime/symbols.internal.h" #include "libc/str/str.h" +#include "libc/thread/thread.h" #define LIMIT 100 @@ -54,7 +57,7 @@ noinstrument noasan int PrintBacktraceUsingSymbols(int fd, struct Garbages *garbage; const struct StackFrame *frame; if (!bp) bp = __builtin_frame_address(0); - garbage = weaken(__garbage); + garbage = __tls_enabled ? ((cthread_t)__get_tls())->garbages : 0; gi = garbage ? garbage->i : 0; for (i = 0, frame = bp; frame; frame = frame->next) { if (++i == LIMIT) { diff --git a/libc/log/log.mk b/libc/log/log.mk index 53db6cbcd..438a19291 100644 --- a/libc/log/log.mk +++ b/libc/log/log.mk @@ -69,6 +69,7 @@ o/$(MODE)/libc/log/watch.o: private \ OVERRIDE_CFLAGS += \ -ffreestanding +o/$(MODE)/libc/log/watch.o \ o/$(MODE)/libc/log/attachdebugger.o \ o/$(MODE)/libc/log/checkaligned.o \ o/$(MODE)/libc/log/checkfail.o \ diff --git a/libc/log/printgarbage.c b/libc/log/printgarbage.c index 69f0bc574..911a1d986 100644 --- a/libc/log/printgarbage.c +++ b/libc/log/printgarbage.c @@ -20,8 +20,11 @@ #include "libc/intrin/kprintf.h" #include "libc/log/log.h" #include "libc/nexgen32e/gc.internal.h" +#include "libc/nexgen32e/gettls.h" +#include "libc/nexgen32e/threaded.h" #include "libc/stdio/stdio.h" -/* clang-format off */ +#include "libc/thread/thread.h" +// clang-format off /** * Prints list of deferred operations on shadow stack. @@ -30,24 +33,25 @@ void PrintGarbage(void) { size_t i; char name[19]; const char *symbol; + struct Garbages *g; kprintf("\n"); kprintf(" SHADOW STACK @ %p\n", __builtin_frame_address(0)); kprintf("garbage ent. parent frame original ret callback arg \n"); kprintf("------------ ------------ ------------------ ------------------ ------------------\n"); - if (__garbage.i) { - for (i = __garbage.i; i--;) { - symbol = __get_symbol_by_addr(__garbage.p[i].ret); + if ((g = __tls_enabled ? ((cthread_t)__get_tls())->garbages:0) && g->i) { + for (i = g->i; i--;) { + symbol = __get_symbol_by_addr(g->p[i].ret); if (symbol) { ksnprintf(name, sizeof(name), "%s", symbol); } else { - ksnprintf(name, sizeof(name), "%#014lx", __garbage.p[i].ret); + ksnprintf(name, sizeof(name), "%#014lx", g->p[i].ret); } kprintf("%12lx %12lx %18s %18s %#18lx\n", - __garbage.p + i, - __garbage.p[i].frame, + g->p + i, + g->p[i].frame, name, - __get_symbol_by_addr(__garbage.p[i].fn), - __garbage.p[i].arg); + __get_symbol_by_addr(g->p[i].fn), + g->p[i].arg); } } else { kprintf("%12s %12s %18s %18s %18s\n","empty","-","-","-","-"); diff --git a/libc/mem/_gc_free.c b/libc/mem/_gc_free.c new file mode 100644 index 000000000..b1679f0ad --- /dev/null +++ b/libc/mem/_gc_free.c @@ -0,0 +1,24 @@ +/*-*- mode:c;indent-tabs-mode:nil;c-basic-offset:2;tab-width:8;coding:utf-8 -*-│ +│vi: set net ft=c ts=2 sts=2 sw=2 fenc=utf-8 :vi│ +╞══════════════════════════════════════════════════════════════════════════════╡ +│ Copyright 2022 Justine Alexandra Roberts Tunney │ +│ │ +│ Permission to use, copy, modify, and/or distribute this software for │ +│ any purpose with or without fee is hereby granted, provided that the │ +│ above copyright notice and this permission notice appear in all copies. │ +│ │ +│ THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL │ +│ WARRANTIES WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED │ +│ WARRANTIES OF MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE │ +│ AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL │ +│ DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR │ +│ PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER │ +│ TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR │ +│ PERFORMANCE OF THIS SOFTWARE. │ +╚─────────────────────────────────────────────────────────────────────────────*/ +#include "libc/mem/mem.h" +#include "libc/runtime/gc.h" + +void _gc_free(void *p) { + free(p); +} diff --git a/libc/mem/defer.greg.c b/libc/mem/defer.greg.c deleted file mode 100644 index 5b2d04a0c..000000000 --- a/libc/mem/defer.greg.c +++ /dev/null @@ -1,85 +0,0 @@ -/*-*- mode:c;indent-tabs-mode:nil;c-basic-offset:2;tab-width:8;coding:utf-8 -*-│ -│vi: set net ft=c ts=2 sts=2 sw=2 fenc=utf-8 :vi│ -╞══════════════════════════════════════════════════════════════════════════════╡ -│ Copyright 2020 Justine Alexandra Roberts Tunney │ -│ │ -│ Permission to use, copy, modify, and/or distribute this software for │ -│ any purpose with or without fee is hereby granted, provided that the │ -│ above copyright notice and this permission notice appear in all copies. │ -│ │ -│ THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL │ -│ WARRANTIES WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED │ -│ WARRANTIES OF MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE │ -│ AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL │ -│ DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR │ -│ PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER │ -│ TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR │ -│ PERFORMANCE OF THIS SOFTWARE. │ -╚─────────────────────────────────────────────────────────────────────────────*/ -#include "libc/assert.h" -#include "libc/intrin/likely.h" -#include "libc/intrin/weaken.h" -#include "libc/calls/calls.h" -#include "libc/mem/mem.h" -#include "libc/nexgen32e/gc.internal.h" -#include "libc/runtime/gc.internal.h" -#include "libc/runtime/runtime.h" -#include "libc/str/str.h" - -forceinline bool PointerNotOwnedByParentStackFrame(struct StackFrame *frame, - struct StackFrame *parent, - void *ptr) { - return !(((intptr_t)ptr > (intptr_t)frame) && - ((intptr_t)ptr < (intptr_t)parent)); -} - -static void __garbage_destroy(void) { - if (weaken(free)) { - weaken(free)(__garbage.p); - } - bzero(&__garbage, sizeof(__garbage)); -} - -void __deferer(struct StackFrame *frame, void *fn, void *arg) { - size_t n2; - struct Garbage *p2; - if (UNLIKELY(__garbage.i == __garbage.n)) { - p2 = __garbage.p; - n2 = __garbage.n + (__garbage.n >> 1); - if (__garbage.p != __garbage.initmem) { - if (!weaken(realloc)) return; - if (!(p2 = weaken(realloc)(p2, n2 * sizeof(*p2)))) return; - } else { - if (!weaken(malloc)) return; - if (!(p2 = weaken(malloc)(n2 * sizeof(*p2)))) return; - memcpy(p2, __garbage.p, __garbage.n * sizeof(*p2)); - atexit(__garbage_destroy); - } - __garbage.p = p2; - __garbage.n = n2; - } - __garbage.p[__garbage.i].frame = frame; - __garbage.p[__garbage.i].fn = (intptr_t)fn; - __garbage.p[__garbage.i].arg = (intptr_t)arg; - __garbage.p[__garbage.i].ret = frame->addr; - __garbage.i++; - frame->addr = (intptr_t)__gc; -} - -/** - * Adds destructor to garbage shadow stack. - * - * @param frame is passed automatically by wrapper macro - * @param fn takes one argument - * @param arg is passed to fn(arg) - * @return arg - */ -void __defer(struct StackFrame *frame, void *fn, void *arg) { - struct StackFrame *f; - if (!arg) return; - f = __builtin_frame_address(0); - assert(__garbage.n); - assert(f->next == frame); - assert(PointerNotOwnedByParentStackFrame(f, frame, arg)); - __deferer(frame, fn, arg); -} diff --git a/libc/mem/gc.c b/libc/mem/gc.c index f53eacec2..0280884c9 100644 --- a/libc/mem/gc.c +++ b/libc/mem/gc.c @@ -16,9 +16,93 @@ │ TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR │ │ PERFORMANCE OF THIS SOFTWARE. │ ╚─────────────────────────────────────────────────────────────────────────────*/ +#include "libc/assert.h" +#include "libc/intrin/kprintf.h" +#include "libc/intrin/likely.h" +#include "libc/mem/mem.h" +#include "libc/nexgen32e/gc.internal.h" +#include "libc/nexgen32e/gettls.h" #include "libc/nexgen32e/stackframe.h" -#include "libc/runtime/gc.h" +#include "libc/nexgen32e/threaded.h" #include "libc/runtime/runtime.h" +#include "libc/str/str.h" +#include "libc/thread/thread.h" + +static inline bool PointerNotOwnedByParentStackFrame(struct StackFrame *frame, + struct StackFrame *parent, + void *ptr) { + return !(((intptr_t)ptr > (intptr_t)frame) && + ((intptr_t)ptr < (intptr_t)parent)); +} + +static void TeardownGc(void) { + int i; + cthread_t tls; + struct Garbages *g; + if (__tls_enabled) { + tls = (cthread_t)__get_tls(); + if ((g = tls->garbages)) { + // exit() currently doesn't use _gclongjmp() like pthread_exit() + // so we need to run the deferred functions manually. + while (g->i) { + --g->i; + ((void (*)(intptr_t))g->p[g->i].fn)(g->p[g->i].arg); + } + free(g->p); + free(g); + } + } +} + +__attribute__((__constructor__)) static void InitializeGc(void) { + atexit(TeardownGc); +} + +// add item to garbage shadow stack. +// then rewrite caller's return address on stack. +static void DeferFunction(struct StackFrame *frame, void *fn, void *arg) { + int n2; + cthread_t tls; + struct Garbage *p2; + struct Garbages *g; + TlsIsRequired(); + tls = (cthread_t)__get_tls(); + g = tls->garbages; + if (UNLIKELY(!g)) { + if (!(g = malloc(sizeof(struct Garbages)))) notpossible; + g->i = 0; + g->n = 4; + if (!(g->p = malloc(g->n * sizeof(struct Garbage)))) { + kprintf("malloc failed\n"); + notpossible; + } + tls->garbages = g; + } else if (UNLIKELY(g->i == g->n)) { + p2 = g->p; + n2 = g->n + (g->n >> 1); + if ((p2 = realloc(p2, n2 * sizeof(*p2)))) { + g->p = p2; + g->n = n2; + } else { + notpossible; + } + } + g->p[g->i].frame = frame; + g->p[g->i].fn = (intptr_t)fn; + g->p[g->i].arg = (intptr_t)arg; + g->p[g->i].ret = frame->addr; + g->i++; + frame->addr = (intptr_t)__gc; +} + +// the gnu extension macros for _gc / _defer point here +void __defer(void *rbp, void *fn, void *arg) { + struct StackFrame *f, *frame = rbp; + f = __builtin_frame_address(0); + assert(f->next == frame); + assert(PointerNotOwnedByParentStackFrame(f, frame, arg)); + DeferFunction(frame, fn, arg); +} /** * Frees memory when function returns. @@ -38,11 +122,12 @@ * @warning do not realloc() with gc()'d pointer * @warning be careful about static keyword due to impact of inlining * @note you should use -fno-omit-frame-pointer + * @threadsafe */ void *(_gc)(void *thing) { struct StackFrame *frame; frame = __builtin_frame_address(0); - __deferer(frame->next, _weakfree, thing); + DeferFunction(frame->next, _weakfree, thing); return thing; } @@ -57,10 +142,11 @@ void *(_gc)(void *thing) { * @warning do not realloc() with gc()'d pointer * @warning be careful about static keyword due to impact of inlining * @note you should use -fno-omit-frame-pointer + * @threadsafe */ void *(_defer)(void *fn, void *arg) { struct StackFrame *frame; frame = __builtin_frame_address(0); - __deferer(frame->next, fn, arg); + DeferFunction(frame->next, fn, arg); return arg; } diff --git a/libc/nexgen32e/gc.S b/libc/nexgen32e/gc.S index d93859808..83143e336 100644 --- a/libc/nexgen32e/gc.S +++ b/libc/nexgen32e/gc.S @@ -20,8 +20,6 @@ #include "libc/dce.h" #include "libc/notice.inc" -#define INITIAL_CAPACITY 4 - nop // Invokes deferred function calls. @@ -34,9 +32,12 @@ // // @param rax,rdx,xmm0,xmm1,st0,st1 is return value // @see test/libc/runtime/gc_test.c -__gc: decq __garbage(%rip) - mov __garbage(%rip),%r8 - mov __garbage+16(%rip),%r9 +// @threadsafe +__gc: mov %fs:0,%rcx # __get_tls() + mov 0x18(%rcx),%rcx # cthread_t::garbages + decl (%rcx) # ++g->i + mov (%rcx),%r8d # r8 = g->i + mov 8(%rcx),%r9 # r9 = g->p js 9f shl $5,%r8 lea (%r9,%r8),%r8 @@ -55,25 +56,5 @@ __gc: decq __garbage(%rip) mov -8(%rbp),%rax leave ret -9: hlt +9: ud2 .endfn __gc,globl,hidden - - .bss - .align 8 -__garbage: - .quad 0 # garbage.i - .quad 0 # garbage.n - .quad 0 # garbage.p - .rept INITIAL_CAPACITY - .quad 0 # garbage.p[𝑖].frame - .quad 0 # garbage.p[𝑖].fn - .quad 0 # garbage.p[𝑖].arg - .quad 0 # garbage.p[𝑖].ret - .endr - .endobj __garbage,globl,hidden - .previous - - .init.start 100,_init_garbage - movb $INITIAL_CAPACITY,__garbage+8(%rip) - movl $__garbage+24,__garbage+16(%rip) - .init.end 100,_init_garbage diff --git a/libc/nexgen32e/gc.internal.h b/libc/nexgen32e/gc.internal.h index 2e621005b..b0adc7ac7 100644 --- a/libc/nexgen32e/gc.internal.h +++ b/libc/nexgen32e/gc.internal.h @@ -12,13 +12,10 @@ struct Garbage { }; struct Garbages { - size_t i, n; + int i, n; struct Garbage *p; - struct Garbage initmem[1]; }; -extern struct Garbages __garbage; - int64_t __gc(void); COSMOPOLITAN_C_END_ diff --git a/libc/nexgen32e/gclongjmp.S b/libc/nexgen32e/gclongjmp.S index 200a3b1eb..4bfdd3755 100644 --- a/libc/nexgen32e/gclongjmp.S +++ b/libc/nexgen32e/gclongjmp.S @@ -28,29 +28,33 @@ // @param esi is returned by setjmp() invocation (coerced nonzero) // @assume system five nexgen32e abi conformant // @see examples/ctrlc.c +// @threadsafe // @noreturn _gclongjmp: push %rbp mov %rsp,%rbp .profilable - lea __garbage(%rip),%r12 - mov (%r12),%r13 # garbage.i - test %r13,%r13 + mov %fs:0,%r12 # __get_tls() + mov 0x18(%r12),%r12 # cthread_t::garbages + test %r12,%r12 + jz 0f + movl (%r12),%r13d # garbages.i + test %r13d,%r13d jnz .L.unwind.destructors 0: jmp longjmp .L.unwind.destructors: push %rdi push %rsi - mov 16(%r12),%r14 # garbage.p + mov 8(%r12),%r14 # garbages.p mov (%rdi),%r15 # jmp_buf[0] is new %rsp shl $5,%r13 # log2(sizeof(struct Garbage)) 1: sub $32,%r13 # 𝑖-- js 2f - cmp (%r14,%r13),%r15 # new %rsp > garbage.p[𝑖].frame + cmp (%r14,%r13),%r15 # new %rsp > garbages.p[𝑖].frame jbe 2f - mov 16(%r14,%r13),%rdi # garbage.p[𝑖].arg - callq *8(%r14,%r13) # garbage.p[𝑖].fn - decq (%r12) # garbage.i-- + mov 16(%r14,%r13),%rdi # garbages.p[𝑖].arg + callq *8(%r14,%r13) # garbages.p[𝑖].fn + decl (%r12) # garbages.i-- jmp 1b 2: pop %rsi pop %rdi diff --git a/libc/nexgen32e/threaded.h b/libc/nexgen32e/threaded.h index 7ecdc6c28..7878eb841 100644 --- a/libc/nexgen32e/threaded.h +++ b/libc/nexgen32e/threaded.h @@ -7,6 +7,7 @@ extern int __threaded; extern bool __tls_enabled; extern unsigned __tls_index; +void TlsIsRequired(void); void *__initialize_tls(char[64]); void __install_tls(char[64]); diff --git a/libc/runtime/enable_tls.c b/libc/runtime/enable_tls.c index d0ec56509..090f3cd95 100644 --- a/libc/runtime/enable_tls.c +++ b/libc/runtime/enable_tls.c @@ -230,7 +230,7 @@ privileged void __enable_tls(void) { // we're checking for the following expression: // 0144 == p[0] && // %fs - // 0110 == p[1] && // rex.w (64-bit operand size) + // 0110 == (p[1] & 0373) && // rex.w (and ignore rex.r) // (0213 == p[2] || // mov reg/mem → reg (word-sized) // 0003 == p[2]) && // add reg/mem → reg (word-sized) // 0004 == (p[3] & 0307) && // mod/rm (4,reg,0) means sib → reg @@ -239,7 +239,7 @@ privileged void __enable_tls(void) { // 0000 == p[6] && // displacement // 0000 == p[7] && // displacement // 0000 == p[8] // displacement - w = READ64LE(p) & READ64LE("\377\377\377\307\377\377\377\377"); + w = READ64LE(p) & READ64LE("\377\373\377\307\377\377\377\377"); if ((w == READ64LE("\144\110\213\004\045\000\000\000") || w == READ64LE("\144\110\003\004\045\000\000\000")) && !p[8]) { diff --git a/libc/runtime/fork-nt.c b/libc/runtime/fork-nt.c index 761999061..a80a6a75b 100644 --- a/libc/runtime/fork-nt.c +++ b/libc/runtime/fork-nt.c @@ -29,6 +29,7 @@ #include "libc/mem/alloca.h" #include "libc/mem/mem.h" #include "libc/nexgen32e/nt2sysv.h" +#include "libc/nexgen32e/threaded.h" #include "libc/nt/console.h" #include "libc/nt/createfile.h" #include "libc/nt/enum/accessmask.h" @@ -88,7 +89,6 @@ static inline textwindows ssize_t ForkIo(int64_t h, char *p, size_t n, static dontinline textwindows bool ForkIo2(int64_t h, void *buf, size_t n, bool32 (*fn)(), const char *sf) { ssize_t rc = ForkIo(h, buf, n, fn); - __tls_enabled = false; // prevent tls crash in kprintf NTTRACE("%s(%ld, %p, %'zu) → %'zd% m", sf, h, buf, n, rc); return rc != -1; } @@ -98,6 +98,7 @@ static dontinline textwindows bool WriteAll(int64_t h, void *buf, size_t n) { } static textwindows dontinline void ReadOrDie(int64_t h, void *buf, size_t n) { + __tls_enabled = false; // prevent tls crash in kprintf if (!ForkIo2(h, buf, n, ReadFile, "ReadFile")) { AbortFork("ReadFile"); } diff --git a/libc/runtime/gc.h b/libc/runtime/gc.h index 5f507652e..03c733fad 100644 --- a/libc/runtime/gc.h +++ b/libc/runtime/gc.h @@ -1,27 +1,24 @@ #ifndef COSMOPOLITAN_LIBC_RUNTIME_GC_H_ #define COSMOPOLITAN_LIBC_RUNTIME_GC_H_ -#include "libc/calls/calls.h" -#include "libc/nexgen32e/stackframe.h" -#include "libc/runtime/runtime.h" #if !(__ASSEMBLER__ + __LINKER__ + 0) COSMOPOLITAN_C_START_ -void *_gc(void *) hidden; -void *_defer(void *, void *) hidden; -void __defer(struct StackFrame *, void *, void *) hidden; -void __deferer(struct StackFrame *, void *, void *) hidden; -void _gclongjmp(jmp_buf, int) dontthrow wontreturn; +void *_gc(void *); +void *_defer(void *, void *); +void __defer(void *, void *, void *); +void _gclongjmp(void *, int) dontthrow wontreturn; +void _gc_free(void *); #if defined(__GNUC__) && !defined(__STRICT_ANSI__) -#define _gc(THING) _defer((void *)_weakfree, (void *)(THING)) -#define _defer(FN, ARG) \ - ({ \ - autotype(ARG) Arg = (ARG); \ - /* prevent weird opts like tail call */ \ - asm volatile("" : "+g"(Arg) : : "memory"); \ - __defer((struct StackFrame *)__builtin_frame_address(0), FN, Arg); \ - asm volatile("" : "+g"(Arg) : : "memory"); \ - Arg; \ +#define _gc(THING) _defer((void *)_gc_free, (void *)(THING)) +#define _defer(FN, ARG) \ + ({ \ + autotype(ARG) Arg = (ARG); \ + /* prevent weird opts like tail call */ \ + asm volatile("" : "+g"(Arg) : : "memory"); \ + __defer(__builtin_frame_address(0), FN, Arg); \ + asm volatile("" : "+g"(Arg) : : "memory"); \ + Arg; \ }) #endif /* defined(__GNUC__) && !defined(__STRICT_ANSI__) */ diff --git a/libc/thread/pthread_create.c b/libc/thread/pthread_create.c index 95972f674..cf138051d 100644 --- a/libc/thread/pthread_create.c +++ b/libc/thread/pthread_create.c @@ -16,6 +16,7 @@ │ TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR │ │ PERFORMANCE OF THIS SOFTWARE. │ ╚─────────────────────────────────────────────────────────────────────────────*/ +#include "libc/assert.h" #include "libc/calls/calls.h" #include "libc/errno.h" #include "libc/intrin/asan.internal.h" @@ -24,7 +25,9 @@ #include "libc/intrin/wait0.internal.h" #include "libc/intrin/weaken.h" #include "libc/mem/mem.h" +#include "libc/nexgen32e/gc.internal.h" #include "libc/nexgen32e/gettls.h" +#include "libc/nexgen32e/threaded.h" #include "libc/runtime/runtime.h" #include "libc/sysv/consts/clone.h" #include "libc/sysv/consts/map.h" @@ -56,6 +59,7 @@ static int PosixThread(void *arg, int tid) { if (weaken(_pthread_key_destruct)) { weaken(_pthread_key_destruct)(0); } + cthread_ungarbage(); if (atomic_load_explicit(&pt->status, memory_order_acquire) == kPosixThreadDetached) { atomic_store_explicit(&pt->status, kPosixThreadZombie, @@ -108,6 +112,7 @@ int pthread_create(pthread_t *thread, const pthread_attr_t *attr, int rc, e = errno; struct PosixThread *pt; pthread_attr_t default_attr; + TlsIsRequired(); _pthread_zombies_decimate(); // default attributes diff --git a/libc/thread/pthread_exit.c b/libc/thread/pthread_exit.c index d33bd8fe6..c6cb898ad 100644 --- a/libc/thread/pthread_exit.c +++ b/libc/thread/pthread_exit.c @@ -18,6 +18,7 @@ ╚─────────────────────────────────────────────────────────────────────────────*/ #include "libc/intrin/pthread.h" #include "libc/nexgen32e/gettls.h" +#include "libc/runtime/gc.h" #include "libc/thread/posixthread.internal.h" #include "libc/thread/spawn.h" #include "libc/thread/thread.h" @@ -38,7 +39,7 @@ wontreturn void pthread_exit(void *rc) { struct PosixThread *pt; if ((pt = ((cthread_t)__get_tls())->pthread)) { pt->rc = rc; - longjmp(pt->exiter, 1); + _gclongjmp(pt->exiter, 1); } else { _Exit1((int)(intptr_t)rc); } diff --git a/libc/thread/spawn.c b/libc/thread/spawn.c index 66b851259..e58f76a59 100644 --- a/libc/thread/spawn.c +++ b/libc/thread/spawn.c @@ -30,6 +30,7 @@ #include "libc/sysv/consts/clone.h" #include "libc/sysv/consts/map.h" #include "libc/sysv/consts/prot.h" +#include "libc/sysv/errfuns.h" #include "libc/thread/spawn.h" #include "libc/thread/thread.h" @@ -54,6 +55,20 @@ #define _TIBZ sizeof(struct cthread_descriptor_t) #define _MEMZ ROUNDUP(_TLSZ + _TIBZ, alignof(struct cthread_descriptor_t)) +struct spawner { + int (*fun)(void *, int); + void *arg; +}; + +static int Spawner(void *arg, int tid) { + int rc; + struct spawner *spawner = arg; + rc = spawner->fun(spawner->arg, tid); + cthread_ungarbage(); + free(spawner); + return 0; +} + /** * Spawns thread, e.g. * @@ -78,6 +93,9 @@ */ int _spawn(int fun(void *, int), void *arg, struct spawn *opt_out_thread) { struct spawn *th, ths; + struct spawner *spawner; + TlsIsRequired(); + if (!fun) return einval(); // we need to to clobber the output memory before calling clone, since // there's no guarantee clone() won't suspend the parent, and focus on @@ -102,11 +120,14 @@ int _spawn(int fun(void *, int), void *arg, struct spawn *opt_out_thread) { return -1; } - if (clone(fun, th->stk, GetStackSize() - 16 /* openbsd:stackbound */, + spawner = malloc(sizeof(struct spawner)); + spawner->fun = fun; + spawner->arg = arg; + if (clone(Spawner, th->stk, GetStackSize() - 16 /* openbsd:stackbound */, CLONE_VM | CLONE_THREAD | CLONE_FS | CLONE_FILES | CLONE_SIGHAND | CLONE_SETTLS | CLONE_PARENT_SETTID | CLONE_CHILD_SETTID | CLONE_CHILD_CLEARTID, - arg, &th->ptid, th->tib, th->ctid) == -1) { + spawner, &th->ptid, th->tib, th->ctid) == -1) { _freestack(th->stk); free(th->tls); return -1; diff --git a/libc/thread/thread.h b/libc/thread/thread.h index 2dce41568..c9bfb5e6a 100644 --- a/libc/thread/thread.h +++ b/libc/thread/thread.h @@ -28,7 +28,7 @@ struct FtraceTls { /* 16 */ struct cthread_descriptor_t { struct cthread_descriptor_t *self; /* 0x00 */ struct FtraceTls ftrace; /* 0x08 */ - void *garbages; /* 0x10 */ + void *garbages; /* 0x18 */ locale_t locale; /* 0x20 */ pthread_t pthread; /* 0x28 */ struct cthread_descriptor_t *self2; /* 0x30 */ @@ -64,6 +64,7 @@ int cthread_sem_wait(cthread_sem_t *, int, const struct timespec *); int cthread_sem_signal(cthread_sem_t *); int cthread_memory_wait32(int *, int, const struct timespec *); int cthread_memory_wake32(int *, int); +void cthread_ungarbage(void); COSMOPOLITAN_C_END_ #endif /* !(__ASSEMBLER__ + __LINKER__ + 0) */ diff --git a/libc/log/printgarbagenumeric.c b/libc/thread/ungarbage.c similarity index 67% rename from libc/log/printgarbagenumeric.c rename to libc/thread/ungarbage.c index 1f6bb4bbc..80de172cd 100644 --- a/libc/log/printgarbagenumeric.c +++ b/libc/thread/ungarbage.c @@ -1,7 +1,7 @@ /*-*- mode:c;indent-tabs-mode:nil;c-basic-offset:2;tab-width:8;coding:utf-8 -*-│ │vi: set net ft=c ts=2 sts=2 sw=2 fenc=utf-8 :vi│ ╞══════════════════════════════════════════════════════════════════════════════╡ -│ Copyright 2021 Justine Alexandra Roberts Tunney │ +│ Copyright 2022 Justine Alexandra Roberts Tunney │ │ │ │ Permission to use, copy, modify, and/or distribute this software for │ │ any purpose with or without fee is hereby granted, provided that the │ @@ -16,27 +16,19 @@ │ TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR │ │ PERFORMANCE OF THIS SOFTWARE. │ ╚─────────────────────────────────────────────────────────────────────────────*/ -#include "libc/log/log.h" +#include "libc/assert.h" +#include "libc/mem/mem.h" #include "libc/nexgen32e/gc.internal.h" -#include "libc/stdio/stdio.h" -/* clang-format off */ +#include "libc/nexgen32e/gettls.h" +#include "libc/thread/thread.h" -/** - * Prints list of deferred operations on shadow stack w/o symbols. - */ -void PrintGarbageNumeric(FILE *f) { - size_t i; - f = stderr; - fprintf(f, "\n"); - fprintf(f, " SHADOW STACK @ 0x%016lx\n", __builtin_frame_address(0)); - fprintf(f, " garbage entry parent frame original ret callback arg \n"); - fprintf(f, "-------------- -------------- ------------------ ------------------ ------------------\n"); - for (i = __garbage.i; i--;) { - fprintf(f, "0x%012lx 0x%012lx 0x%016lx 0x%016lx 0x%016lx\n", - __garbage.p + i, - __garbage.p[i].frame, - __garbage.p[i].ret, - __garbage.p[i].fn, - __garbage.p[i].arg); +void cthread_ungarbage(void) { + struct Garbages *g; + if ((g = ((cthread_t)__get_tls())->garbages)) { + // _pthread_exit() uses _gclongjmp() so if this assertion fails, + // then the likely cause is the thread used gc() with longjmp(). + assert(!g->i); + free(g->p); + free(g); } } diff --git a/test/libc/calls/open_test.c b/test/libc/calls/open_test.c index 4d5b9df89..d7cc6c8ba 100644 --- a/test/libc/calls/open_test.c +++ b/test/libc/calls/open_test.c @@ -16,6 +16,7 @@ │ TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR │ │ PERFORMANCE OF THIS SOFTWARE. │ ╚─────────────────────────────────────────────────────────────────────────────*/ +#include "libc/calls/calls.h" #include "libc/calls/internal.h" #include "libc/dce.h" #include "libc/errno.h" diff --git a/test/libc/nexgen32e/gclongjmp_test.c b/test/libc/nexgen32e/gclongjmp_test.c index a69b0cf22..9b95a37ac 100644 --- a/test/libc/nexgen32e/gclongjmp_test.c +++ b/test/libc/nexgen32e/gclongjmp_test.c @@ -20,12 +20,14 @@ #include "libc/mem/mem.h" #include "libc/nexgen32e/gc.internal.h" #include "libc/nexgen32e/nexgen32e.h" +#include "libc/runtime/gc.h" #include "libc/runtime/gc.internal.h" #include "libc/runtime/runtime.h" #include "libc/stdio/stdio.h" #include "libc/str/str.h" #include "libc/testlib/ezbench.h" #include "libc/testlib/testlib.h" +#include "libc/thread/spawn.h" #include "libc/x/x.h" #define GC(x) _defer(Free, x) @@ -77,6 +79,45 @@ TEST(gclongjmp, test) { free(x); } +void crawl(const char *path) { + const char *dir; + if (!strcmp(path, "/") || !strcmp(path, ".")) return; + crawl(_gc(xdirname(path))); +} + +int Worker(void *arg, int tid) { + crawl("a/b/c/d/a/b/c/d/a/b/c/d/a/b/c/d/a/b/c/d/a/b/c/d"); + return 0; +} + +TEST(gc, torture) { + int i, n = 32; + struct spawn *t = gc(malloc(sizeof(struct spawn) * n)); + for (i = 0; i < n; ++i) ASSERT_SYS(0, 0, _spawn(Worker, 0, t + i)); + for (i = 0; i < n; ++i) EXPECT_SYS(0, 0, _join(t + i)); +} + +void crawl2(jmp_buf jb, const char *path) { + const char *dir; + if (!strcmp(path, "/") || !strcmp(path, ".")) _gclongjmp(jb, 1); + crawl2(jb, _gc(xdirname(path))); +} + +int Worker2(void *arg, int tid) { + jmp_buf jb; + if (!setjmp(jb)) { + crawl2(jb, "a/b/c/d/a/b/c/d/a/b/c/d/a/b/c/d/a/b/c/d/a/b/c/d"); + } + return 0; +} + +TEST(_gclongjmp, torture) { + int i, n = 32; + struct spawn *t = gc(malloc(sizeof(struct spawn) * n)); + for (i = 0; i < n; ++i) ASSERT_SYS(0, 0, _spawn(Worker2, 0, t + i)); + for (i = 0; i < n; ++i) EXPECT_SYS(0, 0, _join(t + i)); +} + dontinline void F1(void) { /* 3x slower than F2() but sooo worth it */ gc(malloc(16)); diff --git a/test/libc/nexgen32e/test.mk b/test/libc/nexgen32e/test.mk index 4728a7cbb..45531fc86 100644 --- a/test/libc/nexgen32e/test.mk +++ b/test/libc/nexgen32e/test.mk @@ -36,6 +36,7 @@ TEST_LIBC_NEXGEN32E_DIRECTDEPS = \ LIBC_STR \ LIBC_STUBS \ LIBC_SYSV \ + LIBC_THREAD \ LIBC_TESTLIB \ LIBC_X \ TOOL_VIZ_LIB \ diff --git a/test/libc/sock/sendrecvmsg_test.c b/test/libc/sock/sendrecvmsg_test.c index c7381a69a..82d6590e1 100644 --- a/test/libc/sock/sendrecvmsg_test.c +++ b/test/libc/sock/sendrecvmsg_test.c @@ -17,6 +17,7 @@ │ PERFORMANCE OF THIS SOFTWARE. │ ╚─────────────────────────────────────────────────────────────────────────────*/ #include "libc/assert.h" +#include "libc/calls/calls.h" #include "libc/calls/struct/iovec.h" #include "libc/dce.h" #include "libc/errno.h" diff --git a/test/libc/sock/test.mk b/test/libc/sock/test.mk index 9b8c60578..a97a9b953 100644 --- a/test/libc/sock/test.mk +++ b/test/libc/sock/test.mk @@ -34,6 +34,7 @@ TEST_LIBC_SOCK_DIRECTDEPS = \ LIBC_STR \ LIBC_STUBS \ LIBC_SYSV \ + LIBC_LOG \ LIBC_SYSV_CALLS \ LIBC_TESTLIB \ LIBC_X \ diff --git a/test/libc/stdio/fgetln_test.c b/test/libc/stdio/fgetln_test.c index 7cba463c6..2f7a00367 100644 --- a/test/libc/stdio/fgetln_test.c +++ b/test/libc/stdio/fgetln_test.c @@ -16,6 +16,7 @@ │ TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR │ │ PERFORMANCE OF THIS SOFTWARE. │ ╚─────────────────────────────────────────────────────────────────────────────*/ +#include "libc/calls/calls.h" #include "libc/mem/mem.h" #include "libc/runtime/gc.internal.h" #include "libc/stdio/stdio.h" diff --git a/third_party/quickjs/call.c b/third_party/quickjs/call.c index 3f087b454..4c293df77 100644 --- a/third_party/quickjs/call.c +++ b/third_party/quickjs/call.c @@ -25,6 +25,7 @@ #include "libc/assert.h" #include "libc/mem/mem.h" #include "libc/runtime/gc.internal.h" +#include "libc/runtime/runtime.h" #include "third_party/quickjs/internal.h" asm(".ident\t\"\\n\\n\ diff --git a/third_party/smallz4/smallz4cat.c b/third_party/smallz4/smallz4cat.c index 2ab09e980..965dce3e7 100644 --- a/third_party/smallz4/smallz4cat.c +++ b/third_party/smallz4/smallz4cat.c @@ -29,6 +29,7 @@ #include "libc/calls/calls.h" #include "libc/mem/mem.h" #include "libc/runtime/gc.internal.h" +#include "libc/runtime/runtime.h" #include "libc/stdio/stdio.h" #include "libc/str/str.h" diff --git a/third_party/tidy/config.c b/third_party/tidy/config.c index 33d6c062d..a31fdd189 100644 --- a/third_party/tidy/config.c +++ b/third_party/tidy/config.c @@ -18,6 +18,7 @@ #include "libc/calls/calls.h" #include "libc/runtime/gc.internal.h" #include "libc/mem/mem.h" +#include "libc/runtime/runtime.h" #include "third_party/tidy/tags.h" #ifdef WINDOWS_OS diff --git a/tool/build/lib/elfwriter.c b/tool/build/lib/elfwriter.c index a092da79d..83d5673a5 100644 --- a/tool/build/lib/elfwriter.c +++ b/tool/build/lib/elfwriter.c @@ -17,6 +17,7 @@ │ PERFORMANCE OF THIS SOFTWARE. │ ╚─────────────────────────────────────────────────────────────────────────────*/ #include "libc/assert.h" +#include "libc/calls/calls.h" #include "libc/elf/def.h" #include "libc/log/check.h" #include "libc/mem/arraylist2.internal.h" diff --git a/tool/build/lib/psk.c b/tool/build/lib/psk.c index f9ff160bc..42aa2aab4 100644 --- a/tool/build/lib/psk.c +++ b/tool/build/lib/psk.c @@ -16,10 +16,12 @@ │ TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR │ │ PERFORMANCE OF THIS SOFTWARE. │ ╚─────────────────────────────────────────────────────────────────────────────*/ +#include "libc/calls/calls.h" #include "libc/calls/struct/stat.h" #include "libc/log/check.h" #include "libc/mem/mem.h" #include "libc/runtime/gc.internal.h" +#include "libc/runtime/runtime.h" #include "libc/sysv/consts/o.h" #include "libc/x/x.h" #include "tool/build/lib/psk.h" diff --git a/tool/hash/hash.mk b/tool/hash/hash.mk index 139ec3d8f..d1b373763 100644 --- a/tool/hash/hash.mk +++ b/tool/hash/hash.mk @@ -22,6 +22,7 @@ TOOL_HASH_DIRECTDEPS = \ LIBC_RUNTIME \ LIBC_STDIO \ LIBC_STR \ + LIBC_MEM \ LIBC_STUBS TOOL_HASH_DEPS := \ diff --git a/tool/viz/printimage.c b/tool/viz/printimage.c index 514f476e9..118a70954 100644 --- a/tool/viz/printimage.c +++ b/tool/viz/printimage.c @@ -21,6 +21,7 @@ #include "dsp/scale/scale.h" #include "dsp/tty/quant.h" #include "dsp/tty/tty.h" +#include "libc/calls/calls.h" #include "libc/calls/ioctl.h" #include "libc/calls/struct/stat.h" #include "libc/calls/struct/winsize.h"