Get garbage collector working on aarch64

Garbage collection will now happen on arm64 when a function returns,
rather than kicking the can down the road to when the process exits.
This change also does some code cleanup and incorporates suggestions

libc/elf/tinyelf.internal.h

@@ -0,0 +1,91 @@
+#ifndef COSMOPOLITAN_LIBC_ELF_TINYELF_INTERNAL_H_
+#define COSMOPOLITAN_LIBC_ELF_TINYELF_INTERNAL_H_
+#include "libc/elf/struct/ehdr.h"
+#include "libc/elf/struct/phdr.h"
+#include "libc/elf/struct/shdr.h"
+#include "libc/elf/struct/sym.h"
+#include "libc/intrin/bits.h"
+#include "libc/limits.h"
+#include "libc/log/libfatal.internal.h"
+#if !(__ASSEMBLER__ + __LINKER__ + 0)
+COSMOPOLITAN_C_START_
+
+#define GetStr(tab, rva)     ((char *)(tab) + (rva))
+#define GetSection(e, s)     ((void *)((intptr_t)(e) + (size_t)(s)->sh_offset))
+#define GetShstrtab(e)       GetSection(e, GetShdr(e, (e)->e_shstrndx))
+#define GetSectionName(e, s) GetStr(GetShstrtab(e), (s)->sh_name)
+#define GetPhdr(e, i)                            \
+  ((Elf64_Phdr *)((intptr_t)(e) + (e)->e_phoff + \
+                  (size_t)(e)->e_phentsize * (i)))
+#define GetShdr(e, i)                            \
+  ((Elf64_Shdr *)((intptr_t)(e) + (e)->e_shoff + \
+                  (size_t)(e)->e_shentsize * (i)))
+
+static inline char *GetStrtab(Elf64_Ehdr *e, size_t *n) {
+  char *name;
+  Elf64_Half i;
+  Elf64_Shdr *shdr;
+  for (i = 0; i < e->e_shnum; ++i) {
+    shdr = GetShdr(e, i);
+    if (shdr->sh_type == SHT_STRTAB) {
+      name = GetSectionName(e, GetShdr(e, i));
+      if (name && READ64LE(name) == READ64LE(".strtab")) {
+        if (n) *n = shdr->sh_size;
+        return GetSection(e, shdr);
+      }
+    }
+  }
+  return 0;
+}
+
+static inline Elf64_Sym *GetSymtab(Elf64_Ehdr *e, Elf64_Xword *n) {
+  Elf64_Half i;
+  Elf64_Shdr *shdr;
+  for (i = e->e_shnum; i > 0; --i) {
+    shdr = GetShdr(e, i - 1);
+    if (shdr->sh_type == SHT_SYMTAB) {
+      if (shdr->sh_entsize != sizeof(Elf64_Sym)) continue;
+      if (n) *n = shdr->sh_size / shdr->sh_entsize;
+      return GetSection(e, shdr);
+    }
+  }
+  return 0;
+}
+
+static inline void GetImageRange(Elf64_Ehdr *elf, intptr_t *x, intptr_t *y) {
+  unsigned i;
+  Elf64_Phdr *phdr;
+  intptr_t start, end, pstart, pend;
+  start = INTPTR_MAX;
+  end = 0;
+  for (i = 0; i < elf->e_phnum; ++i) {
+    phdr = GetPhdr(elf, i);
+    if (phdr->p_type != PT_LOAD) continue;
+    pstart = phdr->p_vaddr;
+    pend = phdr->p_vaddr + phdr->p_memsz;
+    if (pstart < start) start = pstart;
+    if (pend > end) end = pend;
+  }
+  if (x) *x = start;
+  if (y) *y = end;
+}
+
+static inline bool GetElfSymbolValue(const Elf64_Ehdr *ehdr, const char *name,
+                                     uint64_t *res) {
+  Elf64_Xword i, n;
+  const char *stab;
+  const Elf64_Sym *st;
+  if (!(stab = GetStrtab(ehdr, 0))) return false;
+  if (!(st = GetSymtab(ehdr, &n))) return false;
+  for (i = 0; i < n; ++i) {
+    if (!__strcmp(GetStr(stab, st[i].st_name), name)) {
+      *res = st[i].st_value;
+      return true;
+    }
+  }
+  return false;
+}
+
+COSMOPOLITAN_C_END_
+#endif /* !(__ASSEMBLER__ + __LINKER__ + 0) */
+#endif /* COSMOPOLITAN_LIBC_ELF_TINYELF_INTERNAL_H_ */

libc/intrin/bsf.h

@@ -9,26 +9,10 @@ int _bsfll(long long) pureconst;
 int _bsf128(uintmax_t) pureconst;
 
 #if defined(__GNUC__) && !defined(__STRICT_ANSI__)
-#ifdef __x86_64__
-#define _bsf(u)                                                 \
-  ({                                                            \
-    unsigned BiTs;                                              \
-    asm("bsf\t%0,%0" : "=r"(BiTs) : "0"((unsigned)(u)) : "cc"); \
-    BiTs;                                                       \
-  })
-#define _bsfl(u)                                                     \
-  ({                                                                 \
-    unsigned long BiTs;                                              \
-    asm("bsf\t%0,%0" : "=r"(BiTs) : "0"((unsigned long)(u)) : "cc"); \
-    (unsigned)BiTs;                                                  \
-  })
-#define _bsfll(u) _bsfl(u)
-#else
 #define _bsf(x)   __builtin_ctz(x)
 #define _bsfl(x)  __builtin_ctzl(x)
 #define _bsfll(x) __builtin_ctzll(x)
 #endif
-#endif
 
 COSMOPOLITAN_C_END_
 #endif /* !(__ASSEMBLER__ + __LINKER__ + 0) */

libc/intrin/bsr.h

@@ -8,27 +8,10 @@ int _bsrl(long) pureconst;
 int _bsrll(long long) pureconst;
 
 #if defined(__GNUC__) && !defined(__STRICT_ANSI__)
-#ifdef __x86_64__
-int _bsr128(uint128_t) pureconst;
-#define _bsr(u)                                                 \
-  ({                                                            \
-    unsigned BiTs;                                              \
-    asm("bsr\t%0,%0" : "=r"(BiTs) : "0"((unsigned)(u)) : "cc"); \
-    BiTs;                                                       \
-  })
-#define _bsrl(u)                                                     \
-  ({                                                                 \
-    unsigned long BiTs;                                              \
-    asm("bsr\t%0,%0" : "=r"(BiTs) : "0"((unsigned long)(u)) : "cc"); \
-    (unsigned)BiTs;                                                  \
-  })
-#define _bsrll(u) _bsrl(u)
-#else
 #define _bsr(x)   (__builtin_clz(x) ^ (sizeof(int) * CHAR_BIT - 1))
 #define _bsrl(x)  (__builtin_clzl(x) ^ (sizeof(long) * CHAR_BIT - 1))
 #define _bsrll(x) (__builtin_clzll(x) ^ (sizeof(long long) * CHAR_BIT - 1))
 #endif
-#endif
 
 COSMOPOLITAN_C_END_
 #endif /* !(__ASSEMBLER__ + __LINKER__ + 0) */

libc/log/oncrash_arm64.c

@@ -164,6 +164,7 @@ static bool AppendFileLine(struct Buffer *b, const char *addr2line,
 
 relegated void __oncrash_arm64(int sig, struct siginfo *si, void *arg) {
   char buf[10000];
+  ucontext_t *ctx = arg;
   static _Thread_local bool once;
   struct Buffer b[1] = {{buf, sizeof(buf)}};
   b->p[b->i++] = '\n';
@@ -172,7 +173,6 @@ relegated void __oncrash_arm64(int sig, struct siginfo *si, void *arg) {
     const char *kind;
     const char *reset;
     const char *strong;
-    ucontext_t *ctx = arg;
     char host[64] = "unknown";
     struct utsname names = {0};
     once = true;
@@ -320,7 +320,8 @@ relegated void __oncrash_arm64(int sig, struct siginfo *si, void *arg) {
       }
     }
   } else {
-    Append(b, "got %G while crashing!\n", sig);
+    Append(b, "got %G while crashing! pc %lx lr %lx\n", sig,
+           ctx->uc_mcontext.pc, ctx->uc_mcontext.regs[30]);
   }
   sys_write(2, b->p, MIN(b->i, b->n));
   __print_maps();

libc/mem/gc.c

@@ -86,9 +86,7 @@ static void DeferFunction(struct StackFrame *frame, void *fn, void *arg) {
   g->p[g->i].arg = (intptr_t)arg;
   g->p[g->i].ret = frame->addr;
   g->i++;
-#ifdef __x86_64__
   frame->addr = (intptr_t)__gc;
-#endif
 }
 
 // the gnu extension macros for _gc / _defer point here

libc/nexgen32e/gc.S

@@ -33,13 +33,14 @@
 //	@param	rax,rdx,xmm0,xmm1,st0,st1 is return value
 //	@see	test/libc/runtime/gc_test.c
 //	@threadsafe
-__gc:
+	.ftrace1
+__gc:	.ftrace2
 
 #ifdef __x86_64__
 
 	mov	%fs:0,%rcx			// __get_tls()
 	mov	0x18(%rcx),%rcx			// tls::garbages
-	decl	(%rcx)				// ++g->i
+	decl	(%rcx)				// --g->i
 	mov	(%rcx),%r8d			// r8 = g->i
 	mov	8(%rcx),%r9			// r9 = g->p
 	js	9f
@@ -65,18 +66,40 @@ __gc:
 
 #elif defined(__aarch64__)
 
-	stp	x29,x30,[sp,-80]!
+	sub	x8,x28,#1152			// __get_tls()
+	ldr	x9,[x8,0x18]			// tib::garbages
+	ldr	x10,[x9]			// g->i
+	ldr	x8,[x9,8]			// g->p
+	sub	x10,x10,#1			//
+	str	x10,[x9]			//
+	sbfiz	x10,x10,5,32			//
+	add	x8,x8,x10			// g->p+i
+	ldr	x9,[x8,#8]			// g->p[i].fn
+	ldr	x10,[x8,#16]			// g->p[i].arg
+	ldr	x11,[x8,#24]			// g->p[i].ret
+	stp	x29,x11,[sp,-208]!
 	mov	x29,sp
 	stp	x0,x1,[sp,16]
 	stp	x2,x3,[sp,32]
 	stp	x4,x5,[sp,48]
 	stp	x6,x7,[sp,64]
-//	todo jart
-	ldp	x0,x1,[sp,16]
-	ldp	x2,x3,[sp,32]
-	ldp	x4,x5,[sp,48]
+	stp	q0,q1,[sp,80]
+	stp	q2,q3,[sp,112]
+	stp	q4,q5,[sp,144]
+	stp	q6,q7,[sp,176]
+	mov	x1,x10
+	bl	CheckStackIsAligned
+	blr	x9
+	ldp	q6,q7,[sp,176]
+	ldp	q4,q5,[sp,144]
+	ldp	q2,q3,[sp,112]
+	ldp	q0,q1,[sp,80]
 	ldp	x6,x7,[sp,64]
-	ldp	x29,x30,[sp],80
+	ldp	x4,x5,[sp,48]
+	ldp	x2,x3,[sp,32]
+	ldp	x0,x1,[sp,16]
+	ldp	x29,x30,[sp],208
+	ret
 
 #endif /* __x86_64__ */
 

libc/nexgen32e/gclongjmp.S

@@ -36,27 +36,27 @@ _gclongjmp:
 #ifdef __x86_64__
 	push	%rbp
 	mov	%rsp,%rbp
-	mov	%fs:0,%r12			# __get_tls()
-	mov	0x18(%r12),%r12			# Tls::garbages
+	mov	%fs:0,%r12			// __get_tls()
+	mov	0x18(%r12),%r12			// Tls::garbages
 	test	%r12,%r12
 	jz	0f
-	movl	(%r12),%r13d			# garbages.i
+	movl	(%r12),%r13d			// garbages.i
 	test	%r13d,%r13d
 	jnz	.L.unwind.destructors
 0:	jmp	longjmp
 .L.unwind.destructors:
 	push	%rdi
 	push	%rsi
-	mov	8(%r12),%r14			# garbages.p
-	mov	(%rdi),%r15			# jmp_buf[0] is new %rsp
-	shl	$5,%r13				# log2(sizeof(struct Garbage))
-1:	sub	$32,%r13			# 𝑖--
+	mov	8(%r12),%r14			// garbages.p
+	mov	(%rdi),%r15			// jmp_buf[0] is new %rsp
+	shl	$5,%r13				// log2(sizeof(struct Garbage))
+1:	sub	$32,%r13			// 𝑖--
 	js	2f
-	cmp	(%r14,%r13),%r15		# new %rsp > garbages.p[𝑖].frame
+	cmp	(%r14,%r13),%r15		// new %rsp > garbages.p[𝑖].frame
 	jbe	2f
-	mov	16(%r14,%r13),%rdi		# garbages.p[𝑖].arg
-	callq	*8(%r14,%r13)			# garbages.p[𝑖].fn
-	decl	(%r12)				# garbages.i--
+	mov	16(%r14,%r13),%rdi		// garbages.p[𝑖].arg
+	callq	*8(%r14,%r13)			// garbages.p[𝑖].fn
+	decl	(%r12)				// garbages.i--
 	jmp	1b
 2:	pop	%rsi
 	pop	%rdi

libc/nexgen32e/nexgen32e.mk

@@ -60,6 +60,8 @@ o/$(MODE)/libc/nexgen32e/threaded.o: private		\
 			$(NO_MAGIC)
 
 # these assembly files are safe to build on aarch64
+o/$(MODE)/libc/nexgen32e/gc.o: libc/nexgen32e/gc.S
+	@$(COMPILE) -AOBJECTIFY.S $(OBJECTIFY.S) $(OUTPUT_OPTION) -c $<
 o/$(MODE)/libc/nexgen32e/zip.o: libc/nexgen32e/zip.S
 	@$(COMPILE) -AOBJECTIFY.S $(OBJECTIFY.S) $(OUTPUT_OPTION) -c $<
 o/$(MODE)/libc/nexgen32e/mcount.o: libc/nexgen32e/mcount.S

libc/runtime/finddebugbinary.c

@@ -18,10 +18,7 @@
 ╚─────────────────────────────────────────────────────────────────────────────*/
 #include "ape/sections.internal.h"
 #include "libc/calls/calls.h"
-#include "libc/elf/def.h"
-#include "libc/elf/elf.h"
-#include "libc/elf/struct/ehdr.h"
-#include "libc/elf/struct/sym.h"
+#include "libc/elf/tinyelf.internal.h"
 #include "libc/errno.h"
 #include "libc/intrin/bits.h"
 #include "libc/macros.internal.h"
@@ -32,23 +29,6 @@
 #include "libc/sysv/consts/o.h"
 #include "libc/sysv/consts/prot.h"
 
-static bool GetElfSymbolValue(const Elf64_Ehdr *ehdr, size_t esize,
-                              const char *name, uint64_t *res) {
-  Elf64_Xword i, n;
-  const char *stab;
-  const Elf64_Sym *st;
-  if ((stab = GetElfStringTable(ehdr, esize)) &&
-      (st = GetElfSymbolTable(ehdr, esize, &n))) {
-    for (i = 0; i < n; ++i) {
-      if (!strcmp(stab + st[i].st_name, name)) {
-        *res = st[i].st_value;
-        return true;
-      }
-    }
-  }
-  return false;
-}
-
 static bool IsMyDebugBinaryImpl(const char *path) {
   int fd;
   void *map;
@@ -61,8 +41,8 @@ static bool IsMyDebugBinaryImpl(const char *path) {
     // which is currently running in memory.
     if ((size = lseek(fd, 0, SEEK_END)) != -1 &&
         (map = mmap(0, size, PROT_READ, MAP_SHARED, fd, 0)) != MAP_FAILED) {
-      if (IsElf64Binary(map, size) &&
-          GetElfSymbolValue(map, size, "_etext", &value)) {
+      if (READ32LE(map) == READ32LE("\177ELF") &&
+          GetElfSymbolValue(map, "_etext", &value)) {
         res = !_etext || value == (uintptr_t)_etext;
       }
       munmap(map, size);

libc/runtime/opensymboltable.greg.c

@@ -20,11 +20,7 @@
 #include "libc/calls/blockcancel.internal.h"
 #include "libc/calls/calls.h"
 #include "libc/dce.h"
-#include "libc/elf/def.h"
-#include "libc/elf/scalar.h"
-#include "libc/elf/struct/phdr.h"
-#include "libc/elf/struct/shdr.h"
-#include "libc/elf/struct/sym.h"
+#include "libc/elf/tinyelf.internal.h"
 #include "libc/errno.h"
 #include "libc/intrin/bits.h"
 #include "libc/intrin/strace.internal.h"
@@ -41,66 +37,6 @@
 #include "libc/sysv/consts/prot.h"
 #include "libc/sysv/errfuns.h"
 
-#define GetStr(tab, rva)     ((char *)(tab) + (rva))
-#define GetSection(e, s)     ((void *)((intptr_t)(e) + (size_t)(s)->sh_offset))
-#define GetShstrtab(e)       GetSection(e, GetShdr(e, (e)->e_shstrndx))
-#define GetSectionName(e, s) GetStr(GetShstrtab(e), (s)->sh_name)
-#define GetPhdr(e, i)                            \
-  ((Elf64_Phdr *)((intptr_t)(e) + (e)->e_phoff + \
-                  (size_t)(e)->e_phentsize * (i)))
-#define GetShdr(e, i)                            \
-  ((Elf64_Shdr *)((intptr_t)(e) + (e)->e_shoff + \
-                  (size_t)(e)->e_shentsize * (i)))
-
-static char *GetStrtab(Elf64_Ehdr *e, size_t *n) {
-  char *name;
-  Elf64_Half i;
-  Elf64_Shdr *shdr;
-  for (i = 0; i < e->e_shnum; ++i) {
-    shdr = GetShdr(e, i);
-    if (shdr->sh_type == SHT_STRTAB) {
-      name = GetSectionName(e, GetShdr(e, i));
-      if (name && !__strcmp(name, ".strtab")) {
-        if (n) *n = shdr->sh_size;
-        return GetSection(e, shdr);
-      }
-    }
-  }
-  return 0;
-}
-
-static Elf64_Sym *GetSymtab(Elf64_Ehdr *e, Elf64_Xword *n) {
-  Elf64_Half i;
-  Elf64_Shdr *shdr;
-  for (i = e->e_shnum; i > 0; --i) {
-    shdr = GetShdr(e, i - 1);
-    if (shdr->sh_type == SHT_SYMTAB) {
-      if (shdr->sh_entsize != sizeof(Elf64_Sym)) continue;
-      if (n) *n = shdr->sh_size / shdr->sh_entsize;
-      return GetSection(e, shdr);
-    }
-  }
-  return 0;
-}
-
-static void GetImageRange(Elf64_Ehdr *elf, intptr_t *x, intptr_t *y) {
-  unsigned i;
-  Elf64_Phdr *phdr;
-  intptr_t start, end, pstart, pend;
-  start = INTPTR_MAX;
-  end = 0;
-  for (i = 0; i < elf->e_phnum; ++i) {
-    phdr = GetPhdr(elf, i);
-    if (phdr->p_type != PT_LOAD) continue;
-    pstart = phdr->p_vaddr;
-    pend = phdr->p_vaddr + phdr->p_memsz;
-    if (pstart < start) start = pstart;
-    if (pend > end) end = pend;
-  }
-  if (x) *x = start;
-  if (y) *y = end;
-}
-
 static struct SymbolTable *OpenSymbolTableImpl(const char *filename) {
   int fd;
   void *map;