Clean up the TLS code

ape/ape.lds

@@ -180,6 +180,7 @@
 #include "libc/elf/def.h"
 #include "libc/elf/pf2prot.internal.h"
 #include "libc/nt/pedef.internal.h"
+#include "libc/thread/tls.h"
 #include "libc/zip.h"
 
 ENTRY(_start)
@@ -383,17 +384,14 @@ SECTIONS {
     HIDDEN(_ezip = .);
     . = ALIGN(PAGESIZE);
   } :Ram
-  . = ALIGN(PAGESIZE);
 
   .tdata . : {
     _tdata_start = .;
     *(SORT_BY_ALIGNMENT(.tdata))
     *(SORT_BY_ALIGNMENT(.tdata.*))
-    . = ALIGN(16);
     _tdata_end = .;
     . = ALIGN(PAGESIZE);
   } :Tls :Ram
-  . = ALIGN(PAGESIZE);
 
 /*END:   file content that's loaded by o/s */
 /*BEGIN: bss memory void */
@@ -402,7 +400,7 @@ SECTIONS {
     _tbss_start = .;
     *(SORT_BY_ALIGNMENT(.tbss))
     *(SORT_BY_ALIGNMENT(.tbss.*))
-    . = ALIGN(16);
+    . = ALIGN(TLS_ALIGNMENT);
     /* the %fs register is based on this location */
     _tbss_end = .;
   } :Tls
@@ -488,8 +486,10 @@ PFSTUB4(ape_elf_phnum, (ape_phdrs_end - ape_phdrs) / 56);
 PFSTUB4(ape_elf_shnum, 0);
 PFSTUB4(ape_elf_shstrndx, 0);
 
-HIDDEN(_tdata_size = _tdata_end - _tdata_start);
 HIDDEN(_tls_size = _tbss_end - _tdata_start);
+HIDDEN(_tdata_size = _tdata_end - _tdata_start);
+HIDDEN(_tbss_size = _tbss_end - _tbss_start);
+HIDDEN(_tbss_offset = _tbss_start - _tdata_start);
 HIDDEN(_tls_content = (_tdata_end - _tdata_start) + (_tbss_end - _tbss_start));
 
 HIDDEN(__privileged_addr = ROUNDDOWN(__privileged_start, PAGESIZE));
@@ -717,6 +717,9 @@ ASSERT(IS2POW(ape_stack_memsz),
 ASSERT(!(ape_stack_vaddr & (ape_stack_memsz - 1)),
        "ape_stack_vaddr must have ape_stack_memsz alignment; try using STATIC_STACK_ADDR(0x700000000000 - ape_stack_memsz);");
 
+ASSERT(ALIGNOF(.tdata) <= TLS_ALIGNMENT && ALIGNOF(.tbss) <= TLS_ALIGNMENT,
+       "_Thread_local _Alignof can't exceed TLS_ALIGNMENT");
+
 /* Let's not be like Knight Capital. */
 /* NOCROSSREFS_TO(.test .text) */
 

ape/ape.mk

@@ -63,6 +63,7 @@ o/$(MODE)/ape/ape.lds:				\
 		ape/macros.internal.h		\
 		ape/relocations.h		\
 		libc/intrin/bits.h		\
+		libc/thread/tls.h		\
 		libc/calls/struct/timespec.h	\
 		libc/dce.h			\
 		libc/elf/def.h			\
@@ -79,6 +80,7 @@ o/$(MODE)/ape/public/ape.lds:			\
 		ape/macros.internal.h		\
 		ape/relocations.h		\
 		libc/intrin/bits.h		\
+		libc/thread/tls.h		\
 		libc/calls/struct/timespec.h	\
 		libc/dce.h			\
 		libc/elf/def.h			\

libc/runtime/enable_tls.c

@@ -21,6 +21,8 @@
 #include "libc/calls/syscall-sysv.internal.h"
 #include "libc/dce.h"
 #include "libc/errno.h"
+#include "libc/intrin/asan.internal.h"
+#include "libc/intrin/asancodes.h"
 #include "libc/intrin/bits.h"
 #include "libc/intrin/weaken.h"
 #include "libc/log/libfatal.internal.h"
@@ -57,7 +59,7 @@ __msabi extern typeof(TlsAlloc) *const __imp_TlsAlloc;
 
 extern unsigned char __tls_mov_nt_rax[];
 extern unsigned char __tls_add_nt_rax[];
-_Alignas(long) static char __static_tls[5008];
+_Alignas(TLS_ALIGNMENT) static char __static_tls[5008];
 
 /**
  * Enables thread local storage for main process.
@@ -103,7 +105,6 @@ privileged void __enable_tls(void) {
     // if tls requirement is small then use the static tls block
     // which helps avoid a system call for appes with little tls
     // this is crucial to keeping life.com 16 kilobytes in size!
-    _Static_assert(alignof(__static_tls) >= alignof(struct CosmoTib));
     mem = __static_tls;
   } else {
     // if this binary needs a hefty tls block then we'll bank on
@@ -115,6 +116,12 @@ privileged void __enable_tls(void) {
     mem = weaken(_mapanon)(siz);
     assert(mem);
   }
+  if (IsAsan()) {
+    // poison the space between .tdata and .tbss
+    __asan_poison(mem + (intptr_t)_tdata_size,
+                  (intptr_t)_tbss_offset - (intptr_t)_tdata_size,
+                  kAsanProtected);
+  }
   tib = (struct CosmoTib *)(mem + siz - _TIBZ);
   tls = mem + siz - _TIBZ - _TLSZ;
   tib->tib_self = tib;

libc/runtime/internal.h

@@ -23,6 +23,8 @@ extern unsigned char _tdata_end[];
 extern unsigned char _tdata_size[];
 extern unsigned char _tbss_start[];
 extern unsigned char _tbss_end[];
+extern unsigned char _tbss_size[];
+extern unsigned char _tbss_offset[];
 extern unsigned char _tls_size[];
 extern unsigned char _tls_content[];
 

libc/stubs/ld.S

@@ -50,6 +50,8 @@
 	_tdata_size = 0
 	_tbss_start = 0
 	_tbss_end = 0
+	_tbss_offset = 0
+	_tbss_size = 0
 	_tls_size = 0
 	_tls_content = 0
 
@@ -73,6 +75,8 @@
 	.globl	_tdata_size
 	.globl	_tbss_start
 	.globl	_tbss_end
+	.globl	_tbss_size
+	.globl	_tbss_offset
 	.globl	_tls_size
 	.globl	_tls_content
 	.globl	__data_start
@@ -100,8 +104,10 @@
 	.weak	_tdata_size
 	.weak	_tbss_start
 	.weak	_tbss_end
+	.weak	_tbss_size
 	.weak	_tls_size
 	.weak	_tls_content
+	.weak	_tbss_offset
 	.weak	__data_start
 	.weak	__data_end
 	.weak	__bss_start

libc/thread/mktls.c

@@ -16,6 +16,9 @@
 │ TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR             │
 │ PERFORMANCE OF THIS SOFTWARE.                                                │
 ╚─────────────────────────────────────────────────────────────────────────────*/
+#include "libc/dce.h"
+#include "libc/intrin/asan.internal.h"
+#include "libc/intrin/asancodes.h"
 #include "libc/macros.internal.h"
 #include "libc/mem/mem.h"
 #include "libc/runtime/internal.h"
@@ -24,10 +27,9 @@
 #include "libc/thread/spawn.h"
 #include "libc/thread/tls.h"
 
-#define _TLSZ ((intptr_t)_tls_size)
-#define _TLDZ ((intptr_t)_tdata_size)
-#define _TIBZ sizeof(struct CosmoTib)
-#define _MEMZ ROUNDUP(_TLSZ + _TIBZ, _Alignof(struct CosmoTib))
+#define I(x) ((intptr_t)x)
+
+void Bzero(void *, size_t) asm("bzero");  // gcc bug
 
 /**
  * Allocates thread-local storage memory for new thread.
@@ -37,17 +39,25 @@ char *_mktls(char **out_tib) {
   char *tls;
   struct CosmoTib *tib;
 
-  // Allocate enough TLS memory for all the GNU Linuker (_tls_size)
-  // organized _Thread_local data, as well as Cosmpolitan Libc (64)
-  if (!(tls = calloc(1, _MEMZ))) return 0;
+  // allocate memory for tdata, tbss, and tib
+  tls = memalign(TLS_ALIGNMENT, I(_tls_size) + sizeof(struct CosmoTib));
+  if (!tls) return 0;
+
+  // poison memory between tdata and tbss
+  if (IsAsan()) {
+    __asan_poison(tls + I(_tdata_size), I(_tbss_offset) - I(_tdata_size),
+                  kAsanProtected);
+  }
+
+  // initialize tdata and clear tbss
+  memmove(tls, _tdata_start, I(_tdata_size));
+  Bzero(tls + I(_tbss_offset), I(_tbss_size) + sizeof(struct CosmoTib));
 
   // set up thread information block
-  tib = (struct CosmoTib *)(tls + _MEMZ - _TIBZ);
+  tib = (struct CosmoTib *)(tls + I(_tls_size));
   tib->tib_self = tib;
   tib->tib_self2 = tib;
-  tib->tib_errno = 0;
   tib->tib_tid = -1;
-  memmove(tls, _tdata_start, _TLDZ);
 
   if (out_tib) {
     *out_tib = (char *)tib;

libc/thread/tls.h

@@ -1,5 +1,8 @@
 #ifndef COSMOPOLITAN_LIBC_THREAD_TLS_H_
 #define COSMOPOLITAN_LIBC_THREAD_TLS_H_
+
+#define TLS_ALIGNMENT 64
+
 #if !(__ASSEMBLER__ + __LINKER__ + 0)
 COSMOPOLITAN_C_START_
 

test/libc/runtime/tls_test.c

@@ -16,12 +16,39 @@
 │ TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR             │
 │ PERFORMANCE OF THIS SOFTWARE.                                                │
 ╚─────────────────────────────────────────────────────────────────────────────*/
+#include "libc/dce.h"
+#include "libc/intrin/asan.internal.h"
 #include "libc/testlib/testlib.h"
+#include "libc/thread/thread.h"
+#include "libc/thread/tls.h"
 
-_Thread_local int x;
-_Thread_local int y = 40;
-int z = 2;
+#define A TLS_ALIGNMENT
+
+long z = 2;
+pthread_t t;
+_Thread_local long x;
+_Thread_local long y[1] = {40};
+_Alignas(A) _Thread_local long a;
+
+noubsan void *Worker(void *arg) {
+  ASSERT_EQ(42, x + y[0] + z);
+  ASSERT_EQ(0, (intptr_t)&a & (A - 1));
+  if (IsAsan()) {
+    ASSERT_EQ(kAsanProtected, __asan_check(y + 1, sizeof(long)).kind);
+  }
+  return 0;
+}
 
 TEST(tls, test) {
-  EXPECT_EQ(42, x + y + z);
+  ASSERT_EQ(A, _Alignof(a));
+  ASSERT_EQ(0, sizeof(struct CosmoTib) % A);
+  ASSERT_EQ(0, (intptr_t)__get_tls() & (A - 1));
+  EXPECT_EQ(42, x + y[0] + z);
+  y[0] = 666;
+  ASSERT_EQ(0, (intptr_t)&a & (A - 1));
+  ASSERT_EQ(0, pthread_create(&t, 0, Worker, 0));
+  ASSERT_EQ(0, pthread_join(t, 0));
+  if (IsAsan()) {
+    ASSERT_EQ(kAsanProtected, __asan_check(y + 1, sizeof(long)).kind);
+  }
 }