From 333768440c2d6e3b63d30b03eac3595702de2a3c Mon Sep 17 00:00:00 2001 From: Justine Tunney Date: Sat, 10 Sep 2022 11:49:13 -0700 Subject: [PATCH] Clean up the TLS code --- ape/ape.lds | 13 ++++++++----- ape/ape.mk | 2 ++ libc/runtime/enable_tls.c | 11 +++++++++-- libc/runtime/internal.h | 2 ++ libc/stubs/ld.S | 6 ++++++ libc/thread/mktls.c | 30 ++++++++++++++++++++---------- libc/thread/tls.h | 3 +++ test/libc/runtime/tls_test.c | 35 +++++++++++++++++++++++++++++++---- 8 files changed, 81 insertions(+), 21 deletions(-) diff --git a/ape/ape.lds b/ape/ape.lds index 266b8cfb0..6cb55b593 100644 --- a/ape/ape.lds +++ b/ape/ape.lds @@ -180,6 +180,7 @@ #include "libc/elf/def.h" #include "libc/elf/pf2prot.internal.h" #include "libc/nt/pedef.internal.h" +#include "libc/thread/tls.h" #include "libc/zip.h" ENTRY(_start) @@ -383,17 +384,14 @@ SECTIONS { HIDDEN(_ezip = .); . = ALIGN(PAGESIZE); } :Ram - . = ALIGN(PAGESIZE); .tdata . : { _tdata_start = .; *(SORT_BY_ALIGNMENT(.tdata)) *(SORT_BY_ALIGNMENT(.tdata.*)) - . = ALIGN(16); _tdata_end = .; . = ALIGN(PAGESIZE); } :Tls :Ram - . = ALIGN(PAGESIZE); /*END: file content that's loaded by o/s */ /*BEGIN: bss memory void */ @@ -402,7 +400,7 @@ SECTIONS { _tbss_start = .; *(SORT_BY_ALIGNMENT(.tbss)) *(SORT_BY_ALIGNMENT(.tbss.*)) - . = ALIGN(16); + . = ALIGN(TLS_ALIGNMENT); /* the %fs register is based on this location */ _tbss_end = .; } :Tls @@ -488,8 +486,10 @@ PFSTUB4(ape_elf_phnum, (ape_phdrs_end - ape_phdrs) / 56); PFSTUB4(ape_elf_shnum, 0); PFSTUB4(ape_elf_shstrndx, 0); -HIDDEN(_tdata_size = _tdata_end - _tdata_start); HIDDEN(_tls_size = _tbss_end - _tdata_start); +HIDDEN(_tdata_size = _tdata_end - _tdata_start); +HIDDEN(_tbss_size = _tbss_end - _tbss_start); +HIDDEN(_tbss_offset = _tbss_start - _tdata_start); HIDDEN(_tls_content = (_tdata_end - _tdata_start) + (_tbss_end - _tbss_start)); HIDDEN(__privileged_addr = ROUNDDOWN(__privileged_start, PAGESIZE)); @@ -717,6 +717,9 @@ ASSERT(IS2POW(ape_stack_memsz), ASSERT(!(ape_stack_vaddr & (ape_stack_memsz - 1)), "ape_stack_vaddr must have ape_stack_memsz alignment; try using STATIC_STACK_ADDR(0x700000000000 - ape_stack_memsz);"); +ASSERT(ALIGNOF(.tdata) <= TLS_ALIGNMENT && ALIGNOF(.tbss) <= TLS_ALIGNMENT, + "_Thread_local _Alignof can't exceed TLS_ALIGNMENT"); + /* Let's not be like Knight Capital. */ /* NOCROSSREFS_TO(.test .text) */ diff --git a/ape/ape.mk b/ape/ape.mk index 179790eef..02ab10ded 100644 --- a/ape/ape.mk +++ b/ape/ape.mk @@ -63,6 +63,7 @@ o/$(MODE)/ape/ape.lds: \ ape/macros.internal.h \ ape/relocations.h \ libc/intrin/bits.h \ + libc/thread/tls.h \ libc/calls/struct/timespec.h \ libc/dce.h \ libc/elf/def.h \ @@ -79,6 +80,7 @@ o/$(MODE)/ape/public/ape.lds: \ ape/macros.internal.h \ ape/relocations.h \ libc/intrin/bits.h \ + libc/thread/tls.h \ libc/calls/struct/timespec.h \ libc/dce.h \ libc/elf/def.h \ diff --git a/libc/runtime/enable_tls.c b/libc/runtime/enable_tls.c index 0d4ba2ccf..88b6d6e53 100644 --- a/libc/runtime/enable_tls.c +++ b/libc/runtime/enable_tls.c @@ -21,6 +21,8 @@ #include "libc/calls/syscall-sysv.internal.h" #include "libc/dce.h" #include "libc/errno.h" +#include "libc/intrin/asan.internal.h" +#include "libc/intrin/asancodes.h" #include "libc/intrin/bits.h" #include "libc/intrin/weaken.h" #include "libc/log/libfatal.internal.h" @@ -57,7 +59,7 @@ __msabi extern typeof(TlsAlloc) *const __imp_TlsAlloc; extern unsigned char __tls_mov_nt_rax[]; extern unsigned char __tls_add_nt_rax[]; -_Alignas(long) static char __static_tls[5008]; +_Alignas(TLS_ALIGNMENT) static char __static_tls[5008]; /** * Enables thread local storage for main process. @@ -103,7 +105,6 @@ privileged void __enable_tls(void) { // if tls requirement is small then use the static tls block // which helps avoid a system call for appes with little tls // this is crucial to keeping life.com 16 kilobytes in size! - _Static_assert(alignof(__static_tls) >= alignof(struct CosmoTib)); mem = __static_tls; } else { // if this binary needs a hefty tls block then we'll bank on @@ -115,6 +116,12 @@ privileged void __enable_tls(void) { mem = weaken(_mapanon)(siz); assert(mem); } + if (IsAsan()) { + // poison the space between .tdata and .tbss + __asan_poison(mem + (intptr_t)_tdata_size, + (intptr_t)_tbss_offset - (intptr_t)_tdata_size, + kAsanProtected); + } tib = (struct CosmoTib *)(mem + siz - _TIBZ); tls = mem + siz - _TIBZ - _TLSZ; tib->tib_self = tib; diff --git a/libc/runtime/internal.h b/libc/runtime/internal.h index dd4e9877b..2b0e25534 100644 --- a/libc/runtime/internal.h +++ b/libc/runtime/internal.h @@ -23,6 +23,8 @@ extern unsigned char _tdata_end[]; extern unsigned char _tdata_size[]; extern unsigned char _tbss_start[]; extern unsigned char _tbss_end[]; +extern unsigned char _tbss_size[]; +extern unsigned char _tbss_offset[]; extern unsigned char _tls_size[]; extern unsigned char _tls_content[]; diff --git a/libc/stubs/ld.S b/libc/stubs/ld.S index a3570b917..b729f18d2 100644 --- a/libc/stubs/ld.S +++ b/libc/stubs/ld.S @@ -50,6 +50,8 @@ _tdata_size = 0 _tbss_start = 0 _tbss_end = 0 + _tbss_offset = 0 + _tbss_size = 0 _tls_size = 0 _tls_content = 0 @@ -73,6 +75,8 @@ .globl _tdata_size .globl _tbss_start .globl _tbss_end + .globl _tbss_size + .globl _tbss_offset .globl _tls_size .globl _tls_content .globl __data_start @@ -100,8 +104,10 @@ .weak _tdata_size .weak _tbss_start .weak _tbss_end + .weak _tbss_size .weak _tls_size .weak _tls_content + .weak _tbss_offset .weak __data_start .weak __data_end .weak __bss_start diff --git a/libc/thread/mktls.c b/libc/thread/mktls.c index 8b9fb1a3b..8eccb9014 100644 --- a/libc/thread/mktls.c +++ b/libc/thread/mktls.c @@ -16,6 +16,9 @@ │ TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR │ │ PERFORMANCE OF THIS SOFTWARE. │ ╚─────────────────────────────────────────────────────────────────────────────*/ +#include "libc/dce.h" +#include "libc/intrin/asan.internal.h" +#include "libc/intrin/asancodes.h" #include "libc/macros.internal.h" #include "libc/mem/mem.h" #include "libc/runtime/internal.h" @@ -24,10 +27,9 @@ #include "libc/thread/spawn.h" #include "libc/thread/tls.h" -#define _TLSZ ((intptr_t)_tls_size) -#define _TLDZ ((intptr_t)_tdata_size) -#define _TIBZ sizeof(struct CosmoTib) -#define _MEMZ ROUNDUP(_TLSZ + _TIBZ, _Alignof(struct CosmoTib)) +#define I(x) ((intptr_t)x) + +void Bzero(void *, size_t) asm("bzero"); // gcc bug /** * Allocates thread-local storage memory for new thread. @@ -37,17 +39,25 @@ char *_mktls(char **out_tib) { char *tls; struct CosmoTib *tib; - // Allocate enough TLS memory for all the GNU Linuker (_tls_size) - // organized _Thread_local data, as well as Cosmpolitan Libc (64) - if (!(tls = calloc(1, _MEMZ))) return 0; + // allocate memory for tdata, tbss, and tib + tls = memalign(TLS_ALIGNMENT, I(_tls_size) + sizeof(struct CosmoTib)); + if (!tls) return 0; + + // poison memory between tdata and tbss + if (IsAsan()) { + __asan_poison(tls + I(_tdata_size), I(_tbss_offset) - I(_tdata_size), + kAsanProtected); + } + + // initialize tdata and clear tbss + memmove(tls, _tdata_start, I(_tdata_size)); + Bzero(tls + I(_tbss_offset), I(_tbss_size) + sizeof(struct CosmoTib)); // set up thread information block - tib = (struct CosmoTib *)(tls + _MEMZ - _TIBZ); + tib = (struct CosmoTib *)(tls + I(_tls_size)); tib->tib_self = tib; tib->tib_self2 = tib; - tib->tib_errno = 0; tib->tib_tid = -1; - memmove(tls, _tdata_start, _TLDZ); if (out_tib) { *out_tib = (char *)tib; diff --git a/libc/thread/tls.h b/libc/thread/tls.h index f5bf47e47..f5a3f422d 100644 --- a/libc/thread/tls.h +++ b/libc/thread/tls.h @@ -1,5 +1,8 @@ #ifndef COSMOPOLITAN_LIBC_THREAD_TLS_H_ #define COSMOPOLITAN_LIBC_THREAD_TLS_H_ + +#define TLS_ALIGNMENT 64 + #if !(__ASSEMBLER__ + __LINKER__ + 0) COSMOPOLITAN_C_START_ diff --git a/test/libc/runtime/tls_test.c b/test/libc/runtime/tls_test.c index 59e2ad501..69bf48225 100644 --- a/test/libc/runtime/tls_test.c +++ b/test/libc/runtime/tls_test.c @@ -16,12 +16,39 @@ │ TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR │ │ PERFORMANCE OF THIS SOFTWARE. │ ╚─────────────────────────────────────────────────────────────────────────────*/ +#include "libc/dce.h" +#include "libc/intrin/asan.internal.h" #include "libc/testlib/testlib.h" +#include "libc/thread/thread.h" +#include "libc/thread/tls.h" -_Thread_local int x; -_Thread_local int y = 40; -int z = 2; +#define A TLS_ALIGNMENT + +long z = 2; +pthread_t t; +_Thread_local long x; +_Thread_local long y[1] = {40}; +_Alignas(A) _Thread_local long a; + +noubsan void *Worker(void *arg) { + ASSERT_EQ(42, x + y[0] + z); + ASSERT_EQ(0, (intptr_t)&a & (A - 1)); + if (IsAsan()) { + ASSERT_EQ(kAsanProtected, __asan_check(y + 1, sizeof(long)).kind); + } + return 0; +} TEST(tls, test) { - EXPECT_EQ(42, x + y + z); + ASSERT_EQ(A, _Alignof(a)); + ASSERT_EQ(0, sizeof(struct CosmoTib) % A); + ASSERT_EQ(0, (intptr_t)__get_tls() & (A - 1)); + EXPECT_EQ(42, x + y[0] + z); + y[0] = 666; + ASSERT_EQ(0, (intptr_t)&a & (A - 1)); + ASSERT_EQ(0, pthread_create(&t, 0, Worker, 0)); + ASSERT_EQ(0, pthread_join(t, 0)); + if (IsAsan()) { + ASSERT_EQ(kAsanProtected, __asan_check(y + 1, sizeof(long)).kind); + } }