diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 3a2eede21..2b69d7f53 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -4,12 +4,10 @@ on: push: branches: - "master" - - "vista" - "flake" pull_request: branches: - "master" - - "vista" # run workflow manually from the Actions tab workflow_dispatch: diff --git a/Makefile b/Makefile index a8c92725b..de71e9b39 100644 --- a/Makefile +++ b/Makefile @@ -122,7 +122,6 @@ include libc/vga/vga.mk #─┘ include libc/calls/calls.mk #─┐ include libc/runtime/runtime.mk # ├──SYSTEMS RUNTIME include libc/crt/crt.mk # │ You can issue system calls -include third_party/nsync/nsync.mk # │ include third_party/dlmalloc/dlmalloc.mk #─┘ include libc/mem/mem.mk #─┐ include libc/zipos/zipos.mk # ├──DYNAMIC RUNTIME @@ -212,7 +211,6 @@ include test/libc/fmt/test.mk include test/libc/dns/test.mk include test/libc/time/test.mk include test/libc/stdio/test.mk -include test/libc/zipos/test.mk include test/libc/release/test.mk include test/libc/test.mk include test/net/http/test.mk @@ -300,7 +298,6 @@ COSMOPOLITAN_OBJECTS = \ LIBC_MEM \ THIRD_PARTY_DLMALLOC \ LIBC_RUNTIME \ - THIRD_PARTY_NSYNC \ LIBC_ELF \ LIBC_CALLS \ LIBC_SYSV_CALLS \ @@ -340,7 +337,6 @@ COSMOPOLITAN_HEADERS = \ LIBC_RUNTIME \ LIBC_SOCK \ LIBC_STDIO \ - THIRD_PARTY_NSYNC \ THIRD_PARTY_XED \ LIBC_STR \ LIBC_SYSV \ diff --git a/README.md b/README.md index e4f2323c4..0e514a34c 100644 --- a/README.md +++ b/README.md @@ -1,10 +1,7 @@ ![Cosmopolitan Honeybadger](usr/share/img/honeybadger.png) [![build](https://github.com/jart/cosmopolitan/actions/workflows/build.yml/badge.svg)](https://github.com/jart/cosmopolitan/actions/workflows/build.yml) -# Cosmopolitan (Vista Edition) - -**This is a community supported legacy branch of Cosmopolitan 2.0 that's -intended to preserve compatibility with Windows Vista and Windows 7.** +# Cosmopolitan [Cosmopolitan Libc](https://justine.lol/cosmopolitan/index.html) makes C a build-once run-anywhere language, like Java, except it doesn't need an diff --git a/ape/ape.S b/ape/ape.S index 873c1b523..9ff9f4cd9 100644 --- a/ape/ape.S +++ b/ape/ape.S @@ -329,7 +329,7 @@ dsknfo: push %bx movpp %es,%ds xor %si,%si mov %si,%es - mov $0x1d10,%si # mman::pc_drive_base_table + mov $0x1510,%si # mman::pc_drive_base_table xchg %si,%di movsw #→ headunloadtime, headloadtime movsw #→ shutofftime, bytespersector @@ -382,11 +382,11 @@ pcread: push %ax add $512>>4,%si mov %si,%es inc %al # ++sector - cmp 0x1d1c,%al # mman::pc_drive_last_sector + cmp 0x151c,%al # mman::pc_drive_last_sector jbe 2f mov $1,%al inc %dh # ++head - cmp 0x1d20,%dh # mman::pc_drive_last_head + cmp 0x1520,%dh # mman::pc_drive_last_head jbe 2f xor %dh,%dh inc %cx # ++cylinder @@ -485,19 +485,19 @@ ape.mbrpad: ape_disk: .stub ape.diskid,quad .org 0x1be,0x00 - .macro .partn x:req sta h0 s0 c0 fs h9 s9 c9 lba0 nsec - .stub ape.part\x\().status,byte,\sta # 0=non-boot / 0x80=active - .stub ape.part\x\().first.head,byte,\h0 - .stub ape.part\x\().first.sector,byte,\s0 # in low 6 bits - .stub ape.part\x\().first.cylinder,byte,\c0 - .stub ape.part\x\().filesystem,byte,\fs - .stub ape.part\x\().last.head,byte,\h9 - .stub ape.part\x\().last.sector,byte,\s9 - .stub ape.part\x\().last.cylinder,byte,\c9 - .stub ape.part\x\().lba,long,\lba0 # c₀*Cₙ + h₀*Hₙ + s₀*Sₙ - .stub ape.part\x\().sector.count,long,\nsec # sectors are 512 bytes + .macro .partn x + .stub ape.part\x\().status,byte # 0=absent / 0x80=present + .stub ape.part\x\().first.head,byte # in low 6 bits + .stub ape.part\x\().first.cylinder,byte + .stub ape.part\x\().first.sector,byte + .stub ape.part\x\().filesystem,byte + .stub ape.part\x\().last.head,byte + .stub ape.part\x\().last.cylinder,byte + .stub ape.part\x\().last.sector,byte + .stub ape.part\x\().lba,long # c₀*Cₙ + h₀*Hₙ + s₀*Sₙ + .stub ape.part\x\().sector.count,long # sectors are 512 bytes .endm - .partn 1,0x80,0,1,0,0x7f,0xff,0xff,0xff,0,0xffffffff + .partn 1 .partn 2 .partn 3 .partn 4 @@ -569,11 +569,11 @@ apesh: .ascii "'\n#'\"\n" # sixth edition shebang .ascii "t=\"${TMPDIR:-${HOME:-.}}/.ape\"\n" .ascii "[ -x \"$t\" ] || {\n" .ascii "mkdir -p \"${t%/*}\" &&\n" - .ascii "dd if=\"$o\" of=\"$t.$$\" skip=" + .ascii "dd if=\"$o\" of=\"$t.$$\" skip=\"" .shstub ape_loader_dd_skip,2 - .ascii " count=" + .ascii "\" count=\"" .shstub ape_loader_dd_count,2 - .ascii " bs=64 2>/dev/null\n" + .ascii "\" bs=64 2>/dev/null\n" #if SupportsXnu() .ascii "[ -d /Applications ] && " .ascii "dd if=\"$t.$$\"" @@ -637,11 +637,11 @@ apesh: .ascii "'\n#'\"\n" # sixth edition shebang .ascii "dd if=\"$o\"" .ascii " of=\"$o\"" .ascii " bs=8" - .ascii " skip=" + .ascii " skip=\"" .shstub ape_macho_dd_skip,2 - .ascii " count=" + .ascii "\" count=\"" .shstub ape_macho_dd_count,2 - .ascii " conv=notrunc 2>/dev/null\n" + .ascii "\" conv=notrunc 2>/dev/null\n" #endif /* XNU */ .ascii "[ x\"$1\" = x--assimilate ] && exit 0\n" #ifndef APE_NO_MODIFY_SELF @@ -1291,38 +1291,27 @@ lcheck: pushf # check for i8086 / i8088 / i80186 // Gets memory map from BIOS. e820: mov $0x0510>>4,%di # mman::e820 mov %di,%es - xor %di,%di # es:di is destination buffer + xor %edi,%edi # es:di is destination buffer xor %ebx,%ebx # ebx is an api state tracker 1: mov $0xE820,%eax # magic mov $8+8+4+4,%ecx # sizeof(struct SmapEntry) mov $0x534d4150,%edx # magic number - movl $1,8+8+4/*SmapEntry::acpi3*/(%di) # prefill ACPI attributes; - # apparently some buggy BIOSes say - # that they return this field, yet - # do not fill it correctly int $0x15 # ax,bx,cx,dx,di → ax,bx,cx jc 9f # cf = unsupported or abuse cmp %edx,%eax # more magic means success jne 9f test %cx,%cx # discard empty results jz 5f - mov 8/*LODWORD(SmapEntry::size)*/(%di),%eax - or 8+4/*HIDWORD(SmapEntry::size)*/(%di),%eax - jz 5f cmp $8+8+4+1,%cx # discard if ignore flag jb 4f - testb $1/*don't ignore*/,8+8+4/*SmapEntry::acpi3*/(%di) - jz 5f + testb $1/*ignore*/,8+8+4/*SmapEntry::__acpi3*/(%di) + jnz 5f 4: add $8+8+4+4,%di # keep entry 5: test %ebx,%ebx # last entry? jz 8f - cmp $(256-1)*(8+8+4+4),%di + cmp $0x1000,%di jb 1b -8: xor %ax,%ax # add a blank sentinel entry - mov $(8+8)/2,%cx - cld - rep stosw - ret +8: ret 9: mov $REAL(str.e820),%di call rldie .endfn e820 @@ -1392,16 +1381,9 @@ a20: cli // Initializes long mode paging. pinit: push %ds - push %es #define SEG 0x79000 mov $SEG>>4,%ax mov %ax,%ds - mov %ax,%es - xor %di,%di - xor %ax,%ax - mov $(0x7f000-SEG)/2,%cx - cld - rep stosw movl $0x7d000+PAGE_V+PAGE_RW,0x7e000-SEG # PDPT←PML4T (+) movl $0x7c000+PAGE_V+PAGE_RW,0x7e800-SEG # PDPT←PML4T (-) movl $0x7b000+PAGE_V+PAGE_RW,0x7d000-SEG # PDT←PDPT (+) @@ -1410,14 +1392,13 @@ pinit: push %ds movl $0x79000+PAGE_V+PAGE_RW,0x7a000-SEG # PD←PDT (-) mov $512,%cx # PD±2MB mov $PAGE_V+PAGE_RW,%eax - xor %di,%di -0: stosl + xor %si,%si +0: mov %eax,(%si) add $0x1000,%eax - scasl # di += 4 + add $8,%si loop 0b mov $0x7e000,%eax # PML4T←CR3 mov %eax,%cr3 - pop %es pop %ds ret .endfn pinit @@ -1426,7 +1407,7 @@ pinit: push %ds // // @see Intel Manual V3A §4.1.2 golong: cli - lidt 0x1d22 # mman::bad_idt + lidt 0x1522 # mman::bad_idt mov %cr4,%eax or $CR4_PAE|CR4_PGE|CR4_OSFXSR,%eax mov %eax,%cr4 @@ -1444,8 +1425,8 @@ golong: cli // Long mode is long. .code64 -long: xor %eax,%eax - mov $GDT_LONG_DATA,%al +long: push $GDT_LONG_DATA + pop %rax mov %eax,%ds mov %eax,%ss mov %eax,%es diff --git a/ape/ape.lds b/ape/ape.lds index 6cb55b593..266b8cfb0 100644 --- a/ape/ape.lds +++ b/ape/ape.lds @@ -180,7 +180,6 @@ #include "libc/elf/def.h" #include "libc/elf/pf2prot.internal.h" #include "libc/nt/pedef.internal.h" -#include "libc/thread/tls.h" #include "libc/zip.h" ENTRY(_start) @@ -384,14 +383,17 @@ SECTIONS { HIDDEN(_ezip = .); . = ALIGN(PAGESIZE); } :Ram + . = ALIGN(PAGESIZE); .tdata . : { _tdata_start = .; *(SORT_BY_ALIGNMENT(.tdata)) *(SORT_BY_ALIGNMENT(.tdata.*)) + . = ALIGN(16); _tdata_end = .; . = ALIGN(PAGESIZE); } :Tls :Ram + . = ALIGN(PAGESIZE); /*END: file content that's loaded by o/s */ /*BEGIN: bss memory void */ @@ -400,7 +402,7 @@ SECTIONS { _tbss_start = .; *(SORT_BY_ALIGNMENT(.tbss)) *(SORT_BY_ALIGNMENT(.tbss.*)) - . = ALIGN(TLS_ALIGNMENT); + . = ALIGN(16); /* the %fs register is based on this location */ _tbss_end = .; } :Tls @@ -486,10 +488,8 @@ PFSTUB4(ape_elf_phnum, (ape_phdrs_end - ape_phdrs) / 56); PFSTUB4(ape_elf_shnum, 0); PFSTUB4(ape_elf_shstrndx, 0); -HIDDEN(_tls_size = _tbss_end - _tdata_start); HIDDEN(_tdata_size = _tdata_end - _tdata_start); -HIDDEN(_tbss_size = _tbss_end - _tbss_start); -HIDDEN(_tbss_offset = _tbss_start - _tdata_start); +HIDDEN(_tls_size = _tbss_end - _tdata_start); HIDDEN(_tls_content = (_tdata_end - _tdata_start) + (_tbss_end - _tbss_start)); HIDDEN(__privileged_addr = ROUNDDOWN(__privileged_start, PAGESIZE)); @@ -717,9 +717,6 @@ ASSERT(IS2POW(ape_stack_memsz), ASSERT(!(ape_stack_vaddr & (ape_stack_memsz - 1)), "ape_stack_vaddr must have ape_stack_memsz alignment; try using STATIC_STACK_ADDR(0x700000000000 - ape_stack_memsz);"); -ASSERT(ALIGNOF(.tdata) <= TLS_ALIGNMENT && ALIGNOF(.tbss) <= TLS_ALIGNMENT, - "_Thread_local _Alignof can't exceed TLS_ALIGNMENT"); - /* Let's not be like Knight Capital. */ /* NOCROSSREFS_TO(.test .text) */ diff --git a/ape/ape.mk b/ape/ape.mk index 02ab10ded..179790eef 100644 --- a/ape/ape.mk +++ b/ape/ape.mk @@ -63,7 +63,6 @@ o/$(MODE)/ape/ape.lds: \ ape/macros.internal.h \ ape/relocations.h \ libc/intrin/bits.h \ - libc/thread/tls.h \ libc/calls/struct/timespec.h \ libc/dce.h \ libc/elf/def.h \ @@ -80,7 +79,6 @@ o/$(MODE)/ape/public/ape.lds: \ ape/macros.internal.h \ ape/relocations.h \ libc/intrin/bits.h \ - libc/thread/tls.h \ libc/calls/struct/timespec.h \ libc/dce.h \ libc/elf/def.h \ diff --git a/ape/loader.c b/ape/loader.c index 1e0ae6950..08a736709 100644 --- a/ape/loader.c +++ b/ape/loader.c @@ -179,7 +179,7 @@ struct ElfPhdr { extern char ehdr[]; extern char _end[]; -static void *syscall_; +static void *syscall; static char relocated; static struct PathSearcher ps; extern char __syscall_loader[]; @@ -275,7 +275,7 @@ __attribute__((__noreturn__)) static void Exit(int rc, int os) { asm volatile("call\t*%2" : /* no outputs */ : "a"((IsLinux() ? 60 : 1) | (IsXnu() ? 0x2000000 : 0)), "D"(rc), - "rm"(syscall_) + "rm"(syscall) : "memory"); __builtin_unreachable(); } @@ -285,7 +285,7 @@ static void Close(int fd, int os) { asm volatile("call\t*%4" : "=a"(ax), "=D"(di) : "0"((IsLinux() ? 3 : 6) | (IsXnu() ? 0x2000000 : 0)), "1"(fd), - "rm"(syscall_) + "rm"(syscall) : "rcx", "rdx", "rsi", "r8", "r9", "r10", "r11", "memory", "cc"); } @@ -295,7 +295,7 @@ static int Read(int fd, void *data, int size, int os) { asm volatile("call\t*%8" : "=a"(ax), "=D"(di), "=S"(si), "=d"(dx) : "0"((IsLinux() ? 0 : 3) | (IsXnu() ? 0x2000000 : 0)), "1"(fd), - "2"(data), "3"(size), "rm"(syscall_) + "2"(data), "3"(size), "rm"(syscall) : "rcx", "r8", "r9", "r10", "r11", "memory"); return ax; } @@ -306,7 +306,7 @@ static void Write(int fd, const void *data, int size, int os) { asm volatile("call\t*%8" : "=a"(ax), "=D"(di), "=S"(si), "=d"(dx) : "0"((IsLinux() ? 1 : 4) | (IsXnu() ? 0x2000000 : 0)), "1"(fd), - "2"(data), "3"(size), "rm"(syscall_) + "2"(data), "3"(size), "rm"(syscall) : "rcx", "r8", "r9", "r10", "r11", "memory", "cc"); } @@ -315,7 +315,7 @@ static void Execve(const char *prog, char **argv, char **envp, int os) { asm volatile("call\t*%8" : "=a"(ax), "=D"(di), "=S"(si), "=d"(dx) : "0"(59 | (IsXnu() ? 0x2000000 : 0)), "1"(prog), "2"(argv), - "3"(envp), "rm"(syscall_) + "3"(envp), "rm"(syscall) : "rcx", "r8", "r9", "r10", "r11", "memory", "cc"); } @@ -325,7 +325,7 @@ static int Access(const char *path, int mode, int os) { asm volatile("call\t*%7" : "=a"(ax), "=D"(di), "=S"(si), "=d"(dx) : "0"((IsLinux() ? 21 : 33) | (IsXnu() ? 0x2000000 : 0)), - "1"(path), "2"(mode), "rm"(syscall_) + "1"(path), "2"(mode), "rm"(syscall) : "rcx", "r8", "r9", "r10", "r11", "memory", "cc"); return ax; } @@ -338,7 +338,7 @@ static int Msyscall(long p, long n, int os) { } else { asm volatile("call\t*%6" : "=a"(ax), "=D"(di), "=S"(si) - : "0"(37), "1"(p), "2"(n), "rm"(syscall_) + : "0"(37), "1"(p), "2"(n), "rm"(syscall) : "rcx", "rdx", "r8", "r9", "r10", "r11", "memory", "cc"); return ax; } @@ -350,7 +350,7 @@ static int Open(const char *path, int flags, int mode, int os) { asm volatile("call\t*%8" : "=a"(ax), "=D"(di), "=S"(si), "=d"(dx) : "0"((IsLinux() ? 2 : 5) | (IsXnu() ? 0x2000000 : 0)), - "1"(path), "2"(flags), "3"(mode), "rm"(syscall_) + "1"(path), "2"(flags), "3"(mode), "rm"(syscall) : "rcx", "r8", "r9", "r10", "r11", "memory", "cc"); return ax; } @@ -369,7 +369,7 @@ __attribute__((__noinline__)) static long Mmap(long addr, long size, int prot, "pop\t%%r9" : "=a"(ax), "=D"(di), "=S"(si), "=d"(dx), "+r"(flags_), "+r"(fd_), "+r"(off_) - : "rm"(syscall_), + : "rm"(syscall), "0"((IsLinux() ? 9 : IsFreebsd() ? 477 : 197) | @@ -589,7 +589,7 @@ __attribute__((__noreturn__)) static void Spawn(int os, const char *exe, int fd, // since it probably means a userspace program executed this loader // and passed us a custom syscall function earlier. if (Msyscall(code, codesize, os) != -1) { - syscall_ = 0; + syscall = 0; } #if TROUBLESHOOT @@ -600,7 +600,7 @@ __attribute__((__noreturn__)) static void Spawn(int os, const char *exe, int fd, // to extend the behavior of this loader in the future. we don't need // to clear the xmm registers since the ape loader should be compiled // with the -mgeneral-regs-only flag. - register void *r8 asm("r8") = syscall_; + register void *r8 asm("r8") = syscall; asm volatile("xor\t%%eax,%%eax\n\t" "xor\t%%r9d,%%r9d\n\t" "xor\t%%r10d,%%r10d\n\t" @@ -659,10 +659,10 @@ __attribute__((__noreturn__)) void ApeLoader(long di, long *sp, char dl, } // get syscall function pointer - if (handoff && handoff->systemcall) { - syscall_ = handoff->systemcall; + if (handoff && handoff->syscall) { + syscall = handoff->syscall; } else { - syscall_ = __syscall_loader; + syscall = __syscall_loader; } if (handoff) { diff --git a/ape/loader.h b/ape/loader.h index 45a7ae2b1..7080f1370 100644 --- a/ape/loader.h +++ b/ape/loader.h @@ -14,7 +14,7 @@ struct ApeLoader { int os; char *prog; char *page; - void *systemcall; + void *syscall; }; #endif /* COSMOPOLITAN_APE_LOADER_H_ */ diff --git a/ape/macros.internal.h b/ape/macros.internal.h index c17344116..973602a60 100644 --- a/ape/macros.internal.h +++ b/ape/macros.internal.h @@ -221,41 +221,44 @@ /** * Binary coded decimal support. * - *

This allows linker scripts to generate dd commands, e.g. ape.lds. - * There are a few ways to pad each number to the necessary 8 bytes. - * Spaces cannot be prepended because busybox refuses to parse them. - * Zeros cannot be prepended because Mac will take numbers as octal. - * That leaves appending spaces. The user's shell ought to treat any - * unquoted run of spaces as if there was only one, so this is safe. + *

This allows linker scripts to generate dd commands. Leading spaces + * need to be inserted so Mac doesn't consider them octal; therefore, + * parameters must be quoted; and eight digits should be good enough. */ -#define SHSTUB2(SYM, X) \ - HIDDEN(SYM##_bcs0 = BCD_LEFT(X)); \ - HIDDEN(SYM##_bcs1 = BCD_RIGHT(X)) -#define BCD_SMEAR(X) ((X) + (X) * 10000) -#define BCD_LEFT(X) \ - (((X)) < 10000 ? BCD_RIGHT(BCD_SMEAR(X)) | 0x10 \ - : (X) < 100000 ? BCD_RIGHT(BCD_SMEAR((X) / 10)) \ - : (X) < 1000000 ? BCD_RIGHT(BCD_SMEAR((X) / 100)) \ - : (X) < 10000000 ? BCD_RIGHT(BCD_SMEAR((X) / 1000)) \ - : (X) < 100000000 ? BCD_RIGHT(BCD_SMEAR((X) / 10000)) \ - : 0xffffffffffffffff) -#define BCD_RIGHT(X) \ - (((X)) < 10000 ? 0x20202020 \ - : (X) < 100000 ? 0x20202030 + \ - (X) % 10 \ - : (X) < 1000000 ? 0x20203030 + \ - ((X) / 10) % 10 + \ - (X) % 10 * 0x100 \ - : (X) < 10000000 ? 0x20303030 + \ - ((X) / 100) % 10 + \ - ((X) / 10) % 10 * 0x100 + \ - (X) % 10 * 0x10000 \ - : (X) < 100000000 ? 0x30303030 + \ - ((X) / 1000) % 10 + \ - ((X) / 100) % 10 * 0x100 + \ - ((X) / 10) % 10 * 0x10000 + \ - (X) % 10 * 0x1000000 \ - : 0xffffffffffffffff) +#define SHSTUB2(SYM, X) \ + HIDDEN(SYM##_bcs0 = BCD10K(X)); \ + HIDDEN(SYM##_bcs1 = BCD(X)) +#define BCD(X) \ + ((X) == 0 \ + ? 0x20202030 \ + : (X) < 10 ? 0x30202020 + (((X) % 10) << 24) \ + : (X) < 100 ? 0x30302020 + (((X) % 10) << 24) + \ + (((X) / 10 % 10) << 16) \ + : (X) < 1000 ? 0x30303020 + (((X) % 10) << 24) + \ + (((X) / 10 % 10) << 16) + \ + (((X) / 100 % 10) << 8) \ + : 0x30303030 + (((X) % 10) << 24) + \ + (((X) / 10 % 10) << 16) + \ + (((X) / 100 % 10) << 8) + \ + (((X) / 1000 % 10) << 0)) +#define BCD10K(X) \ + ((X) < 10000 \ + ? 0x20202020 \ + : (X) < 100000 \ + ? 0x30202020 + (((X) / 10000 % 10) << 24) \ + : (X) < 1000000 \ + ? 0x30302020 + (((X) / 10000 % 10) << 24) + \ + (((X) / 100000 % 10) << 16) \ + : (X) < 10000000 \ + ? 0x30303020 + (((X) / 10000 % 10) << 24) + \ + (((X) / 100000 % 10) << 16) + \ + (((X) / 1000000 % 10) << 8) \ + : (X) < 100000000 \ + ? 0x30303030 + (((X) / 10000 % 10) << 24) + \ + (((X) / 100000 % 10) << 16) + \ + (((X) / 1000000 % 10) << 8) + \ + (((X) / 10000000 % 10) << 0) \ + : 0xffffffffffffffff) #endif /* __ASSEMBLER__ */ #endif /* APE_MACROS_H_ */ diff --git a/examples/examples.mk b/examples/examples.mk index d18fe533d..1f1134f09 100644 --- a/examples/examples.mk +++ b/examples/examples.mk @@ -80,7 +80,6 @@ EXAMPLES_DIRECTDEPS = \ THIRD_PARTY_LUA \ THIRD_PARTY_MBEDTLS \ THIRD_PARTY_MUSL \ - THIRD_PARTY_NSYNC \ THIRD_PARTY_QUICKJS \ THIRD_PARTY_STB \ THIRD_PARTY_XED \ diff --git a/examples/greenbean.c b/examples/greenbean.c index 484fa361c..7169a6180 100644 --- a/examples/greenbean.c +++ b/examples/greenbean.c @@ -8,7 +8,6 @@ ╚─────────────────────────────────────────────────────────────────*/ #endif #include "libc/assert.h" -#include "libc/atomic.h" #include "libc/calls/calls.h" #include "libc/calls/struct/sigaction.h" #include "libc/calls/struct/sigset.h" @@ -19,11 +18,14 @@ #include "libc/fmt/itoa.h" #include "libc/intrin/atomic.h" #include "libc/intrin/kprintf.h" +#include "libc/intrin/pthread.h" +#include "libc/intrin/wait0.internal.h" #include "libc/limits.h" #include "libc/log/check.h" #include "libc/log/log.h" #include "libc/macros.internal.h" #include "libc/mem/mem.h" +#include "libc/nexgen32e/threaded.h" #include "libc/runtime/internal.h" #include "libc/runtime/runtime.h" #include "libc/runtime/stack.h" @@ -46,9 +48,6 @@ #include "libc/sysv/consts/sol.h" #include "libc/sysv/consts/tcp.h" #include "libc/thread/spawn.h" -#include "libc/thread/thread.h" -#include "libc/thread/tls.h" -#include "libc/thread/wait0.internal.h" #include "libc/time/struct/tm.h" #include "libc/time/time.h" #include "net/http/http.h" @@ -105,11 +104,11 @@ "Cache-Control: private; max-age=0\r\n" int threads; -atomic_int workers; -atomic_int messages; -atomic_int listening; -atomic_int connections; -atomic_int closingtime; +_Atomic(int) workers; +_Atomic(int) messages; +_Atomic(int) listening; +_Atomic(int) connections; +_Atomic(int) closingtime; const char *volatile status; void *Worker(void *id) { diff --git a/examples/hello4.c b/examples/hello4.c new file mode 100644 index 000000000..f3aecca99 --- /dev/null +++ b/examples/hello4.c @@ -0,0 +1,20 @@ +#if 0 +/*─────────────────────────────────────────────────────────────────╗ +│ To the extent possible under law, Justine Tunney has waived │ +│ all copyright and related or neighboring rights to this file, │ +│ as it is written in the following disclaimers: │ +│ • http://unlicense.org/ │ +│ • http://creativecommons.org/publicdomain/zero/1.0/ │ +╚─────────────────────────────────────────────────────────────────*/ +#endif +#include "libc/math.h" +#include "libc/stdio/stdio.h" + +STATIC_YOINK("vga_console"); + +int main(int argc, char *argv[]) { + volatile long double x = -.5; + volatile long double y = 1.5; + printf("Hello World! %.19Lg\n", atan2l(x, y)); + return 0; +} diff --git a/examples/mu_test.c b/examples/mu_test.c deleted file mode 100644 index e69aa6614..000000000 --- a/examples/mu_test.c +++ /dev/null @@ -1,2054 +0,0 @@ -/*-*- mode:c;indent-tabs-mode:t;c-basic-offset:8;tab-width:8;coding:utf-8 -*-│ -│vi: set et ft=c ts=8 tw=8 fenc=utf-8 :vi│ -╞══════════════════════════════════════════════════════════════════════════════╡ -│ Copyright 2016 Google Inc. │ -│ │ -│ Licensed under the Apache License, Version 2.0 (the "License"); │ -│ you may not use this file except in compliance with the License. │ -│ You may obtain a copy of the License at │ -│ │ -│ http://www.apache.org/licenses/LICENSE-2.0 │ -│ │ -│ Unless required by applicable law or agreed to in writing, software │ -│ distributed under the License is distributed on an "AS IS" BASIS, │ -│ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. │ -│ See the License for the specific language governing permissions and │ -│ limitations under the License. │ -╚─────────────────────────────────────────────────────────────────────────────*/ -#include "libc/calls/calls.h" -#include "libc/calls/weirdtypes.h" -#include "libc/errno.h" -#include "libc/fmt/conv.h" -#include "libc/fmt/fmt.h" -#include "libc/limits.h" -#include "libc/mem/mem.h" -#include "libc/runtime/runtime.h" -#include "libc/stdio/stdio.h" -#include "libc/stdio/temp.h" -#include "libc/str/str.h" -#include "libc/thread/thread.h" -#include "third_party/nsync/atomic.internal.h" -#include "third_party/nsync/cv.h" -#include "third_party/nsync/dll.h" -#include "third_party/nsync/mu.h" -#include "third_party/nsync/mu_wait.h" -#include "third_party/nsync/races.internal.h" -#include "third_party/nsync/time.h" -// clang-format off - -/** - * @fileoverview *NSYNC Mutex Tests / Benchmarks. - * - * make -j8 o//examples/mu_test.com - * o//examples/mu_test.com -b - * - */ - -typedef struct testing_base_s *testing_base; -typedef const struct testing_base_s *const_testing_base; -typedef struct testing_s *testing; - -/* Return a newly initialized testing_base. */ -testing_base testing_new (int argc, char *argv[], int flags); - -/* Return the index of the first argument in argv[] not processed by testing_new() */ -int testing_base_argn (testing_base tb); - -/* exit() with the test's exit status */ -int testing_base_exit (testing_base tb); - -/* Stop and start the benchmark timer. */ -void testing_stop_timer (testing t); -void testing_start_timer (testing t); - -/* Return whether the machine appears to be a uniprocessor. - Some tests get different results on uniprocessors, because - the probability of certain interleavings of thread actions is - greatly reduced. */ -int testing_is_uniprocessor (testing t); - -/* Given a testing_base, run f (t), where t has type testing. - Output will be for a test. */ -#define TEST_RUN(tb, f) testing_run_ ((tb), &f, #f, 0) - -/* Given a testing_base, run f (t), where t has type testing. - Output will be for a benchmark, which should iterate testing_n (t) times. */ -#define BENCHMARK_RUN(tb, f) testing_run_ ((tb), &f, #f, 1) - -/* Return the iteration count for a benchmark. */ -int testing_n (testing t); - -/* Output nul-terminated string msg[] to stderr, then abort(). */ -void testing_panic (const char *msg); - -/* Return a value below 0 if tests should run short, 0 if normal, and a value exceeding - 0 if tests should run long. */ -int testing_longshort (testing t); - -/* Return non-zero if the user requested verbose output. */ -int testing_verbose (testing t); - -/* Output a printf-formated log message associated with *t. - Example: TEST_LOG (t, ("wombat %d", some_int)); - The TEST_ERROR() and TEST_FATAL() forms of the call makr the test as failing. - The TEST_FATAL() form causes other subtests not to run. */ -#define TEST_LOG(t, args) testing_error_ ((t), 0, __FILE__, __LINE__, smprintf args); -#define TEST_ERROR(t, args) testing_error_ ((t), 1, __FILE__, __LINE__, smprintf args); -#define TEST_FATAL(t, args) testing_error_ ((t), 2, __FILE__, __LINE__, smprintf args); - -/* ---------------------------------------- */ - -/* internal details follow */ - -/* An internal routine used by TEST_RUN() and BENCHMARK_RUN(). */ -void testing_run_ (testing_base tb, void (*f) (testing t), const char *name, int is_benchmark); - -/* Output an error message msg, and record status. */ -void testing_error_ (testing t, int status, const char *file, int line, char *msg); - -//////////////////////////////////////////////////////////////////////////////// -// time_extra.h - -/* Return a malloced nul-terminated string representing time t, using - "decimals" decimal places. */ -char *nsync_time_str (nsync_time t, int decimals); - -/* Sleep until the specified time. Returns 0 on success, and EINTR - if the call was interrupted. */ -int nsync_time_sleep_until (nsync_time abs_deadline); - -/* Return t as a double. */ -double nsync_time_to_dbl (nsync_time t); - -/* Return a time corresponding to double d. */ -nsync_time nsync_time_from_dbl (double d); - -//////////////////////////////////////////////////////////////////////////////// -// smprintf.h - -char *smprintf (const char *fmt, ...); - -//////////////////////////////////////////////////////////////////////////////// -// smprintf.c - -char *smprintf (const char *fmt, ...) { - int m = strlen (fmt) * 2 + 1; - char *buf = (char *) malloc (m); - int didnt_fit; - do { - va_list ap; - int x; - va_start (ap, fmt); - x = vsnprintf (buf, m, fmt, ap); - va_end (ap); - if (x >= m) { - buf = (char *) realloc (buf, m = x+1); - didnt_fit = 1; - } else if (x < 0 || x == m-1) { - buf = (char *) realloc (buf, m *= 2); - didnt_fit = 1; - } else { - didnt_fit = 0; - } - } while (didnt_fit); - return (buf); -} - -//////////////////////////////////////////////////////////////////////////////// -// time_extra.c - -char *nsync_time_str (nsync_time t, int decimals) { - static const struct { - const char *suffix; - double multiplier; - } scale[] = { - { "ns", 1.0e-9, }, - { "us", 1e-6, }, - { "ms", 1e-3, }, - { "s", 1.0, }, - { "hr", 3600.0, }, - }; - double s = nsync_time_to_dbl (t); - int i = 0; - while (i + 1 != sizeof (scale) / sizeof (scale[0]) && scale[i + 1].multiplier <= s) { - i++; - } - return (smprintf ("%.*f%s", decimals, s/scale[i].multiplier, scale[i].suffix)); -} - -int nsync_time_sleep_until (nsync_time abs_deadline) { - int result = 0; - nsync_time now; - now = nsync_time_now (); - if (nsync_time_cmp (abs_deadline, now) > 0) { - nsync_time remaining; - remaining = nsync_time_sleep (nsync_time_sub (abs_deadline, now)); - if (nsync_time_cmp (remaining, nsync_time_zero) > 0) { - result = EINTR; - } - } - return (result); -} - -double nsync_time_to_dbl (nsync_time t) { - return (((double) NSYNC_TIME_SEC (t)) + ((double) NSYNC_TIME_NSEC (t) * 1e-9)); -} - -nsync_time nsync_time_from_dbl (double d) { - time_t s = (time_t) d; - if (d < s) { - s--; - } - return (nsync_time_s_ns (s, (unsigned) ((d - (double) s) * 1e9))); -} - -//////////////////////////////////////////////////////////////////////////////// -// start_thread.c - -struct thd_args { - void (*f) (void *); - void *arg; -}; - -static void *body (void *v) { - struct thd_args *args = (struct thd_args *) v; - (*args->f) (args->arg); - free (args); - return (NULL); -} - -void nsync_start_thread_ (void (*f) (void *), void *arg) { - struct thd_args *args = (struct thd_args *) malloc (sizeof (*args)); - pthread_t t; - args->f = f; - args->arg = arg; - pthread_create (&t, NULL, body, args); - pthread_detach (t); -} - -//////////////////////////////////////////////////////////////////////////////// -// closure.h - -/* A run-once, self-freeing closure. */ -typedef struct closure_s { - void (*f0) (void *); -} closure; - -/* Run the closure *cl, and free it. */ -void closure_run (closure *cl); - -/* Fork a new thread running the closure *cl, which will be freed when the - thread exits. */ -void closure_fork (closure *cl); - -/* To create a closure, declare a closure constructor with the right function arguments. - - For functions taking no arguments, use - CLOSURE_DECL_BODY0 (foo) - to generate the static routine: - static closure *closure_foo (void (*f) (void)); - that will return a closure for any function *f that takes no argument. - - For an 1-argument function, use - CLOSURE_DECL_BODY1 (foo, type) - to generate the static routine: - static closure *closure_foo (void (*f) (type), type x); - that will return a closure for any function taking a single argument of the - specified type. - - For an 2-argument function, use - CLOSURE_DECL_BODY2 (foo, type0, type1) - to generate the static routine: - static closure *closure_foo (void (*f) (type0, type1), type0 x0, type1 x1); - that will return a closure for any function taking a "type0" argument, and - a "type1" argument. - - And so on, up to 9 arguments. - - For example, to make closures out of qsort(): - - // First, just once (per module) define: - // static closure *closure_qsort_args ( - // void (*f) (void *, size_t, size_t, int (*)(const void *, const void *)) - // void *x0, size_t x1, size_t x2, int (*x3)(const void *, const void *)); - // by writing: - CLOSURE_DECL_BODY4 (qsort_args, void *, size_t, size_t, int (*)(const void *, const void *)) - - // Second, for each closure to be created, write something like this: - closure *cl = closure_qsort_args (array, n_elements, sizeof (array[0]), &elem_cmp); - - // Then to run (and free) each closure: - closure_run (cl); - // This is like calling - // qsort (array, n_elements, sizeof (array[0]), &elem_cmp); - // free (cl); - */ - - - - -/* ------------------------------------------------------------------ */ -/* Internal macro details follow. */ -#define CLOSURE_S0(x,e) e -#define CLOSURE_S1(x,e) x##0 -#define CLOSURE_S2(x,e) x##0, x##1 -#define CLOSURE_S3(x,e) x##0, x##1, x##2 -#define CLOSURE_S4(x,e) x##0, x##1, x##2, x##3 -#define CLOSURE_S5(x,e) x##0, x##1, x##2, x##3, x##4 -#define CLOSURE_S6(x,e) x##0, x##1, x##2, x##3, x##4, x##5 -#define CLOSURE_S7(x,e) x##0, x##1, x##2, x##3, x##4, x##5, x##6 -#define CLOSURE_S8(x,e) x##0, x##1, x##2, x##3, x##4, x##5, x##6, x##7 -#define CLOSURE_S9(x,e) x##0, x##1, x##2, x##3, x##4, x##5, x##6, x##7, x##8 - -#define CLOSURE_P0(x,y,p,s,t) -#define CLOSURE_P1(x,y,p,s,t) p x##0 y##0 t -#define CLOSURE_P2(x,y,p,s,t) p x##0 y##0 s x##1 y##1 t -#define CLOSURE_P3(x,y,p,s,t) p x##0 y##0 s x##1 y##1 s x##2 y##2 t -#define CLOSURE_P4(x,y,p,s,t) p x##0 y##0 s x##1 y##1 s x##2 y##2 s x##3 y##3 t -#define CLOSURE_P5(x,y,p,s,t) p x##0 y##0 s x##1 y##1 s x##2 y##2 s x##3 y##3 s x##4 y##4 t -#define CLOSURE_P6(x,y,p,s,t) p x##0 y##0 s x##1 y##1 s x##2 y##2 s x##3 y##3 s x##4 y##4 s \ - x##5 y##5 t -#define CLOSURE_P7(x,y,p,s,t) p x##0 y##0 s x##1 y##1 s x##2 y##2 s x##3 y##3 s x##4 y##4 s \ - x##5 y##5 s x##6 y##6 t -#define CLOSURE_P8(x,y,p,s,t) p x##0 y##0 s x##1 y##1 s x##2 y##2 s x##3 y##3 s x##4 y##4 s \ - x##5 y##5 s x##6 y##6 s x##7 y##7 t -#define CLOSURE_P9(x,y,p,s,t) p x##0 y##0 s x##1 y##1 s x##2 y##2 s x##3 y##3 s x##4 y##4 s \ - x##5 y##5 s x##6 y##6 s x##7 y##7 s x##8 y##8 t - -#define CLOSURE_COMMA_ , -#define CLOSURE_SEMI_ ; -#define CLOSURE_BLANK_ - -#define CLOSURE_DECL_BODY_N_(name, n) \ - struct closure_s_##name { \ - void (*f0) (void *); /* must be first; matches closure. */ \ - void (*f) (CLOSURE_S##n (closure_t_##name##_,void)); \ - CLOSURE_P##n (closure_t_##name##_, a, CLOSURE_BLANK_, \ - CLOSURE_SEMI_, CLOSURE_SEMI_) \ - }; \ - static void closure_f0_##name (void *v) { \ - struct closure_s_##name *a = (struct closure_s_##name *) v; \ - (*a->f) (CLOSURE_S##n (a->a,CLOSURE_BLANK_)); \ - free (a); \ - } \ - static closure *closure_##name (void (*f) (CLOSURE_S##n (closure_t_##name##_,void)) \ - CLOSURE_P##n (closure_t_##name##_, a, CLOSURE_COMMA_, \ - CLOSURE_COMMA_, CLOSURE_BLANK_)) { \ - struct closure_s_##name *cl = (struct closure_s_##name *) malloc (sizeof (*cl)); \ - cl->f0 = &closure_f0_##name; \ - cl->f = f; \ - CLOSURE_P##n (cl->a, = a, CLOSURE_BLANK_, CLOSURE_SEMI_, CLOSURE_SEMI_) \ - return ((closure *) cl); \ - } - - -#define CLOSURE_DECL_BODY0(name) \ - CLOSURE_DECL_BODY_N_ (name, 0) -#define CLOSURE_DECL_BODY1(name, t0) \ - typedef t0 closure_t_##name##_0; \ - CLOSURE_DECL_BODY_N_ (name, 1) -#define CLOSURE_DECL_BODY2(name, t0, t1) \ - typedef t0 closure_t_##name##_0; \ - typedef t1 closure_t_##name##_1; \ - CLOSURE_DECL_BODY_N_ (name, 2) -#define CLOSURE_DECL_BODY3(name, t0, t1, t2) \ - typedef t0 closure_t_##name##_0; \ - typedef t1 closure_t_##name##_1; \ - typedef t2 closure_t_##name##_2; \ - CLOSURE_DECL_BODY_N_ (name, 3) -#define CLOSURE_DECL_BODY4(name, t0, t1, t2, t3) \ - typedef t0 closure_t_##name##_0; \ - typedef t1 closure_t_##name##_1; \ - typedef t2 closure_t_##name##_2; \ - typedef t3 closure_t_##name##_3; \ - CLOSURE_DECL_BODY_N_ (name, 4) -#define CLOSURE_DECL_BODY5(name, t0, t1, t2, t3, t4) \ - typedef t0 closure_t_##name##_0; \ - typedef t1 closure_t_##name##_1; \ - typedef t2 closure_t_##name##_2; \ - typedef t3 closure_t_##name##_3; \ - typedef t4 closure_t_##name##_4; \ - CLOSURE_DECL_BODY_N_ (name, 5) -#define CLOSURE_DECL_BODY6(name, t0, t1, t2, t3, t4, t5) \ - typedef t0 closure_t_##name##_0; \ - typedef t1 closure_t_##name##_1; \ - typedef t2 closure_t_##name##_2; \ - typedef t3 closure_t_##name##_3; \ - typedef t4 closure_t_##name##_4; \ - typedef t5 closure_t_##name##_5; \ - CLOSURE_DECL_BODY_N_ (name, 6) -#define CLOSURE_DECL_BODY7(name, t0, t1, t2, t3, t4, t5, t6) \ - typedef t0 closure_t_##name##_0; \ - typedef t1 closure_t_##name##_1; \ - typedef t2 closure_t_##name##_2; \ - typedef t3 closure_t_##name##_3; \ - typedef t4 closure_t_##name##_4; \ - typedef t5 closure_t_##name##_5; \ - typedef t6 closure_t_##name##_6; \ - CLOSURE_DECL_BODY_N_ (name, 7) -#define CLOSURE_DECL_BODY8(name, t0, t1, t2, t3, t4, t5, t6, t7) \ - typedef t0 closure_t_##name##_0; \ - typedef t1 closure_t_##name##_1; \ - typedef t2 closure_t_##name##_2; \ - typedef t3 closure_t_##name##_3; \ - typedef t4 closure_t_##name##_4; \ - typedef t5 closure_t_##name##_5; \ - typedef t6 closure_t_##name##_6; \ - typedef t7 closure_t_##name##_7; \ - CLOSURE_DECL_BODY_N_ (name, 8) -#define CLOSURE_DECL_BODY9(name, t0, t1, t2, t3, t4, t5, t6, t7, t8) \ - typedef t0 closure_t_##name##_0; \ - typedef t1 closure_t_##name##_1; \ - typedef t2 closure_t_##name##_2; \ - typedef t3 closure_t_##name##_3; \ - typedef t4 closure_t_##name##_4; \ - typedef t5 closure_t_##name##_5; \ - typedef t6 closure_t_##name##_6; \ - typedef t7 closure_t_##name##_7; \ - typedef t8 closure_t_##name##_8; \ - CLOSURE_DECL_BODY_N_ (name, 9) - -//////////////////////////////////////////////////////////////////////////////// -// closure.c - -void nsync_start_thread_ (void (*f) (void *), void *arg); - -/* Run the closure *cl. */ -void closure_run (closure *cl) { - (*cl->f0) (cl); -} - -/* Run the closure (closure *), but wrapped to fix the type. */ -static void closure_run_body (void *v) { - closure_run ((closure *)v); -} - -void closure_fork (closure *cl) { - nsync_start_thread_ (&closure_run_body, cl); -} - -//////////////////////////////////////////////////////////////////////////////// -// atm_log.h - -void nsync_atm_log_ (int c, void *p, uint32_t o, uint32_t n, const char *file, int line); -void nsync_atm_log_print_ (void); - -//////////////////////////////////////////////////////////////////////////////// -// atm_log.c - -#ifndef NSYNC_ATM_LOG -#define NSYNC_ATM_LOG 0 -#endif - -struct atm_log { - uintptr_t i; - uintptr_t thd_id; - uintptr_t c; - void *p; - uintptr_t o; - uintptr_t n; - const char *file; - uintptr_t line; -}; - -#define LOG_N 14 - -static struct atm_log log_entries[1 << LOG_N]; -static uint32_t log_i; - -static pthread_mutex_t log_mu; - -static pthread_key_t key; -static pthread_once_t once = PTHREAD_ONCE_INIT; -static void do_once (void) { - pthread_mutex_init (&log_mu, NULL); - pthread_key_create (&key, NULL); -} -static int thread_id; - -void nsync_atm_log_ (int c, void *p, uint32_t o, uint32_t n, const char *file, int line) { - if (NSYNC_ATM_LOG) { - struct atm_log *e; - uint32_t i; - int *pthd_id; - int thd_id; - - pthread_once (&once, &do_once); - pthd_id = (int *) pthread_getspecific (key); - - pthread_mutex_lock (&log_mu); - i = log_i++; - if (pthd_id == NULL) { - thd_id = thread_id++; - pthd_id = (int *) malloc (sizeof (*pthd_id)); - pthread_setspecific (key, pthd_id); - *pthd_id = thd_id; - } else { - thd_id = *pthd_id; - } - pthread_mutex_unlock (&log_mu); - - e = &log_entries[i & ((1 << LOG_N) - 1)]; - e->i = i; - e->thd_id = thd_id; - e->c = c; - e->p = p; - e->o = o; - e->n = n; - e->file = file; - e->line = line; - } -} - -void nsync_atm_log_print_ (void) { - if (NSYNC_ATM_LOG) { - uint32_t i; - pthread_once (&once, &do_once); - pthread_mutex_lock (&log_mu); - for (i = 0; i != (1 << LOG_N); i++) { - struct atm_log *e = &log_entries[i]; - if (e->file != 0) { - fprintf (stderr, "%6lx %3d %c p %16p o %8x n %8x %10s:%d\n", - (unsigned long) e->i, - (int) e->thd_id, - e->c <= ' '? '?' : (char)e->c, - e->p, - (uint32_t) e->o, - (uint32_t) e->n, - e->file, - (int) e->line); - } - } - pthread_mutex_unlock (&log_mu); - } -} - -//////////////////////////////////////////////////////////////////////////////// -// testing.c - -struct testing_base_s { - int flags; /* flags from testing_new(); r/o after init */ - int parallelism; /* max parallelism to use; r/o after init */ - FILE *fp; /* where to send output; pointer is r/o after init */ - int argn; /* first arg not processed by testing_new(); r/o after init */ - int argc; /* argc passed to testing_new(); r/o after init */ - char **argv; /* argv passed to testing_new(); r/o after init */ - char *prog; /* name of programme, from argv[0] */ - int suppress_header; /* supress hreader on benchmarks */ - int run_tests; /* whether to run tests */ - int run_benchmarks; /* whether to run benchmarks */ - int benchmarks_running; /* and benchmarks are now running */ - char *include_pat; /* ,- or |-separated substrings of tests to include */ - char *exclude_pat; /* ,- or |-separated substrings of tests to exclude */ - int longshort; /* 0 normal, -1 short, 1 long */ - int verbose; /* 0 normal; 1 verbose output */ - - nsync_mu testing_mu; /* protects fields below */ - int is_uniprocessor; /* whether the system is a uniprocessor */ - nsync_dll_list_ children; /* list of testing_s structs whose base is this testing_base_s */ - int child_count; /* count of testing_s structs whose base is this testing_base_s */ - int exit_status; /* final exit status */ -}; - -struct testing_s { - struct testing_base_s *base; /* r/o after init */ - int test_status; /* status; merged into common->exit_status */ - int n; /* benchmark iteration count */ - nsync_atomic_uint32_ partial_line; /* whether partial test banner emitted last*/ - FILE *fp; /* where to output; merged into common->fp if != to it */ - nsync_time start_time; /* timer start time; for benchmarks */ - nsync_time stop_time; /* when the timer was stopped; for benchmarks */ - void (*f) (testing); /* test function to run */ - const char *name; /* name of test */ - nsync_dll_element_ siblings; /* part of list of siblings */ -}; - -/* Output the header for benchmarks. */ -static void output_header (FILE *fp, const char *prog) { - int i; - int hdrlen = fprintf (fp, "%-10s%-40s %9s %8s %8s %8s\n", "Benchmark", prog, "ops", "time", - "ops/sec", "time/op"); - for (i = 1; i < hdrlen; i++) { - putc ('-', fp); - } - putc ('\n', fp); - fflush (fp); -} - -/* Process a single flag. *pargn is main's argn */ -static void process_flag (testing_base tb, int *pargn, int argc, char *argv[], int flag, - const char *arg) { - switch (flag) { - case 'b': - tb->run_benchmarks = 1; - break; - case 'B': - tb->run_benchmarks = 1; - tb->run_tests = 0; - break; - case 'H': - output_header (stdout, ""); - exit (0); - case 'h': - tb->suppress_header = 1; - break; - case 'l': - tb->longshort++; - break; - case 'm': - if (*pargn + 1 == argc) { - fprintf (stderr, "%s: -m flag expects ,- or |-separated strings\n", - argv[0]); - exit (2); - } - tb->include_pat = argv[++*pargn]; - break; - case 'n': - if (*pargn + 1 == argc || atoi (argv[1 + *pargn]) < 1) { - fprintf (stderr, "%s: -n flag expects parallelism value >= 1\n", argv[0]); - exit (2); - } - tb->parallelism = atoi (argv[++*pargn]); - break; - case 's': - tb->longshort--; - break; - case 'v': - tb->verbose = 1; - break; - case 'x': - if (*pargn + 1 == argc) { - fprintf (stderr, "%s: -x flag expects ,- or |-separated strings\n", - argv[0]); - exit (2); - } - tb->exclude_pat = argv[++*pargn]; - break; - default: - fprintf (stderr, "%s: unrecognized flag '%c' in arg %d: \"%s\"\n", argv[0], flag, - *pargn, arg); - exit (2); - } -} - -testing_base testing_new (int argc, char *argv[], int flags) { - static const char sep[] = { '/', '\\' }; - int i; - int argn; - testing_base tb = (testing_base)malloc (sizeof (*tb)); - memset ((void *) tb, 0, sizeof (*tb)); - tb->flags = flags; - tb->fp = stderr; - tb->argc = argc; - tb->argv = argv; - tb->parallelism = 1; - tb->run_tests = 1; - tb->is_uniprocessor = -1; - tb->prog = tb->argv[0]; - for (i = 0; i != sizeof (sep) / sizeof (sep[0]); i++) { - char *last = strrchr (tb->prog, sep[i]); - if (last != NULL) { - tb->prog = last + 1; - } - } - for (argn = 1; argn != argc && argv[argn][0] == '-' && - strcmp (argv[argn], "--") != 0; argn++) { - const char *arg = argv[argn]; - const char *f; - for (f = &arg[1]; *f != 0; f++) { - process_flag (tb, &argn, argc, argv, *f, arg); - } - } - tb->argn = argn + (argn != argc && strcmp (argv[argn], "--") == 0); - return (tb); -} - -int testing_verbose (testing t) { - return (t->base->verbose); -} - -int testing_longshort (testing t) { - return (t->base->longshort); -} - -int testing_n (testing t) { - return (t->n); -} - -int testing_base_argn (testing_base tb) { - return (tb->argn); -} - -/* Return whether *(int *)v is zero. Used with nsync_mu_wait(). */ -static int int_is_zero (const void *v) { - return (*(const int *)v == 0); -} - -int testing_base_exit (testing_base tb) { - int exit_status; - nsync_mu_lock (&tb->testing_mu); - nsync_mu_wait (&tb->testing_mu, &int_is_zero, &tb->child_count, NULL); - exit_status = tb->exit_status; - nsync_mu_unlock (&tb->testing_mu); - free (tb); - exit (exit_status); - return (exit_status); -} - -/* Cleanup code used after running either a test or a benchmark, - called at the end of run_test() and run_benchmark(). */ -static void finish_run (testing t) { - testing_base tb = t->base; - fflush (t->fp); - nsync_mu_lock (&tb->testing_mu); - if (t->fp != tb->fp) { - int c; - rewind (t->fp); - while ((c = getc (t->fp)) != EOF) { - putc (c, tb->fp); - } - fclose (t->fp); - fflush (tb->fp); - } - if (tb->exit_status < t->test_status) { - tb->exit_status = t->test_status; - } - tb->children = nsync_dll_remove_ (tb->children, &t->siblings); - tb->child_count--; - nsync_mu_unlock (&tb->testing_mu); - free (t); -} - -/* Run the test (*t->f)(t), and report on t->fp how long it took and its final - status, which is set to non-zero if the test reported errors. */ -static void run_test (testing t) { - testing_base tb = t->base; - char *elapsed_str = 0; - fprintf (t->fp, "%-25s %-45s ", tb->prog, t->name); - fflush (t->fp); - ATM_STORE (&t->partial_line, 1); - t->test_status = 0; - t->n = 0; - t->stop_time = nsync_time_zero; - t->start_time = nsync_time_now (); - (*t->f) (t); - elapsed_str = nsync_time_str (nsync_time_sub (nsync_time_now (), t->start_time), 2); - if (!ATM_LOAD (&t->partial_line)) { - fprintf (t->fp, "%-25s %-45s %s %8s\n", tb->prog, t->name, - t->test_status != 0? "failed": "passed", elapsed_str); - } else { - fprintf (t->fp, "%s %8s\n", t->test_status != 0? "failed": "passed", elapsed_str); - } - ATM_STORE (&t->partial_line, 0); - fflush (t->fp); - free (elapsed_str); - finish_run (t); -} - -/* Run the benchmark (*t->f)(t) repeatedly, specifying successively greater - numbers of iterations, measuring how long it takes each time. Eventually, - it takes long enough to get a reasonable estimate of how long each iteration - takes, which is reported on t->fp. */ -static void run_benchmark (testing t) { - char *elapsed_str = 0; - char *time_per_op_str = 0; - double elapsed; - int n = 1; - double target = 2.0; - int longshort = testing_longshort (t); - if (longshort < 0) { - target = 1e-3 * (2000 >> -longshort); - } else if (longshort > 0) { - target = 1e-3 * (2000 << longshort); - } - do { - int32_t mul; - t->test_status = 0; - t->n = n; - t->stop_time = nsync_time_zero; - t->start_time = nsync_time_now (); - (*t->f) (t); - elapsed = nsync_time_to_dbl (nsync_time_sub (nsync_time_now (), t->start_time)); - if (elapsed < 1e-1) { - elapsed = 1e-1; - } - mul = (int32_t) (target / elapsed); - while (elapsed * mul * 4 < target * 5) { - mul++; - } - if (mul > 1 && elapsed * mul * 2 < target * 3 && n < INT_MAX / mul) { - n *= mul; - } else if (n < INT_MAX / 2) { - n *= 2; - } - } while (t->test_status == 0 && elapsed < target && n != t->n); - elapsed_str = nsync_time_str (nsync_time_from_dbl (elapsed), 2); - time_per_op_str = nsync_time_str (nsync_time_from_dbl (elapsed / t->n), 2); - fprintf (t->fp, "%-50s %9d %8s %8.2g %8s%s\n", t->name, t->n, elapsed_str, - ((double)t->n) / elapsed, time_per_op_str, - t->test_status != 0 ? " *** failed ***" : ""); - free (elapsed_str); - free (time_per_op_str); - finish_run (t); -} - -CLOSURE_DECL_BODY1 (testing, testing) - -/* Return whether there's a "spare thread"; that is, whether the current count - of child threads is less than the allowed parallelism. */ -static int spare_thread (const void *v) { - const_testing_base tb = (const_testing_base) v; - return (tb->child_count < tb->parallelism); -} - -/* Return whether nul-terminated string str[] contains a string listed in - comma-separated (or vertical bar-separted) strings in nul-terminated string - pat[]. A dollar at the end of a string in pat[] matches the end of - string in str[]. */ -static int match (const char *pat, const char *str) { - static const char seps[] = ",|"; - int found = 0; - char Xbuf[128]; - int m = sizeof (Xbuf) - 1; - char *mbuf = NULL; - char *buf = Xbuf; - int i = 0; - while (!found && pat[i] != '\0') { - int blen = strcspn (&pat[i], seps); - int e = i + blen; - if (blen > m) { - m = blen + 128; - buf = mbuf = (char *) realloc (mbuf, m + 1); - } - memcpy (buf, &pat[i], blen); - buf[blen] = '\0'; - if (blen > 0 && buf[blen - 1] == '$') { - int slen = strlen (str); - buf[--blen] = 0; - found = (slen >= blen && - strcmp (&str[slen-blen], buf) == 0); - } else { - found = (strstr (str, buf) != NULL); - } - i = e + strspn (&pat[e], seps); - } - free (mbuf); - return (found); -} - -void testing_run_ (testing_base tb, void (*f) (testing t), const char *name, int is_benchmark) { - int exit_status; - nsync_mu_lock (&tb->testing_mu); - exit_status = tb->exit_status; - nsync_mu_unlock (&tb->testing_mu); - if (exit_status < 2 && - (!is_benchmark || tb->run_benchmarks) && - (is_benchmark || tb->run_tests) && - (tb->include_pat == NULL || match (tb->include_pat, name)) && - (tb->exclude_pat == NULL || !match (tb->exclude_pat, name))) { - testing t = (testing) malloc (sizeof (*t)); - memset ((void *) t, 0, sizeof (*t)); - nsync_dll_init_ (&t->siblings, t); - t->base = tb; - t->f = f; - t->name = name; - if (tb->parallelism == 1) { - t->fp = tb->fp; - } else { - t->fp = tmpfile (); - } - if (!is_benchmark) { - if (tb->benchmarks_running) { - nsync_mu_lock (&tb->testing_mu); - nsync_mu_wait (&tb->testing_mu, &int_is_zero, &tb->child_count, NULL); - nsync_mu_unlock (&tb->testing_mu); - tb->benchmarks_running = 0; - } - nsync_mu_lock (&tb->testing_mu); - nsync_mu_wait (&tb->testing_mu, &spare_thread, tb, NULL); - tb->child_count++; - tb->children = nsync_dll_make_last_in_list_ (tb->children, &t->siblings); - nsync_mu_unlock (&tb->testing_mu); - closure_fork (closure_testing (&run_test, t)); - } else { - if (!tb->benchmarks_running) { - nsync_mu_lock (&tb->testing_mu); - nsync_mu_wait (&tb->testing_mu, &int_is_zero, &tb->child_count, NULL); - nsync_mu_unlock (&tb->testing_mu); - if (!tb->suppress_header) { - output_header (tb->fp, tb->prog); - } - tb->benchmarks_running = 1; - } - nsync_mu_lock (&tb->testing_mu); - nsync_mu_wait (&tb->testing_mu, &spare_thread, tb, NULL); - tb->child_count++; - tb->children = nsync_dll_make_last_in_list_ (tb->children, &t->siblings); - nsync_mu_unlock (&tb->testing_mu); - closure_fork (closure_testing (&run_benchmark, t)); - } - } -} - -/* Used to decide whether the test is running on a uniprocessor. */ -struct is_uniprocessor_s { - double count; /* count of iterations while *state==1 */ - nsync_atomic_uint32_ done; /* set to 1 when thread finishes */ - char dummy[256]; /* so structs don't share cache line */ -}; - -/* An anciliary thread that waits until *state is 1, then increments s->count - while *state stays 1, and then sets s->done to 1 before exiting. */ -static void uniprocessor_check (nsync_atomic_uint32_ *state, struct is_uniprocessor_s *s) { - IGNORE_RACES_START (); - while (ATM_LOAD_ACQ (state) != 1) { - } - while (ATM_LOAD_ACQ (state) == 1) { - s->count++; - } - ATM_STORE_REL (&s->done, 1); - IGNORE_RACES_END (); -} - -CLOSURE_DECL_BODY2 (uniprocessor_check, nsync_atomic_uint32_ *, struct is_uniprocessor_s *) - -/* Return whether the test is running on a uniprocessor. - - Some of the tests rely on interleaving of actions between threads. - Particular interleavings are much less likely on uniprocessors, so the tests - do not run, or do not declare an error if the system is a uniprocessor. - - Operating systems vary significantly in how one may ask how many procerssors - are present, so we use a heuristic based on comparing the timing of a single - thread, and two concurrent threads. */ -int testing_is_uniprocessor (testing t) { - int is_uniprocessor; - nsync_mu_lock (&t->base->testing_mu); - is_uniprocessor = t->base->is_uniprocessor; - if (is_uniprocessor == -1) { - int i; - struct is_uniprocessor_s s[3]; - nsync_atomic_uint32_ state; - for (i = 0; i != 3; i++) { - s[i].count = 0.0; - s[i].done = 0; - } - - ATM_STORE_REL (&state, 0); - closure_fork (closure_uniprocessor_check (&uniprocessor_check, &state, &s[0])); - nsync_time_sleep (nsync_time_ms (100)); - ATM_STORE_REL (&state, 1); - nsync_time_sleep (nsync_time_ms (400)); - ATM_STORE_REL (&state, 2); - while (!ATM_LOAD_ACQ (&s[0].done)) { - } - - ATM_STORE_REL (&state, 0); - closure_fork (closure_uniprocessor_check (&uniprocessor_check, &state, &s[1])); - closure_fork (closure_uniprocessor_check (&uniprocessor_check, &state, &s[2])); - nsync_time_sleep (nsync_time_ms (100)); - ATM_STORE_REL (&state, 1); - nsync_time_sleep (nsync_time_ms (400)); - ATM_STORE_REL (&state, 2); - while (!ATM_LOAD_ACQ (&s[1].done) || !ATM_LOAD_ACQ (&s[2].done)) { - } - t->base->is_uniprocessor = ((s[1].count + s[2].count) / s[0].count) < 1.7; - is_uniprocessor = t->base->is_uniprocessor; - } - nsync_mu_unlock (&t->base->testing_mu); - return (is_uniprocessor); -} - -void testing_stop_timer (testing t) { - if (nsync_time_cmp (t->stop_time, nsync_time_zero) != 0) { - abort (); - } - t->stop_time = nsync_time_now (); -} - -void testing_start_timer (testing t) { - if (nsync_time_cmp (t->stop_time, nsync_time_zero) == 0) { - abort (); - } - t->start_time = nsync_time_add (t->start_time, - nsync_time_sub (nsync_time_now (), t->stop_time)); - t->stop_time = nsync_time_zero; -} - -void testing_error_ (testing t, int test_status, const char *file, int line, char *msg) { - int len = strlen (msg); - int addnl = (len == 0 || msg[len - 1] != '\n'); - if (t->test_status < test_status) { - t->test_status = test_status; - } - if (ATM_LOAD (&t->partial_line)) { - ATM_STORE (&t->partial_line, 0); - fprintf (t->fp, "\n%s: %s:%d: %s%s", - test_status == 2? "fatal": test_status == 1? "error": "info", file, line, msg, - addnl? "\n": ""); - } else { - fprintf (t->fp, "%s: %s:%d: %s%s", - test_status == 2? "fatal": test_status == 1? "error": "info", file, line, msg, - addnl? "\n": ""); - } - free (msg); -} - -void nsync_panic_(const char *); - -/* Abort after printing the nul-terminated string s[]. */ -void testing_panic (const char *s) { - nsync_atm_log_print_ (); - nsync_panic_ (s); -} - -//////////////////////////////////////////////////////////////////////////////// -// mu_test.c - -/* The state shared between the threads in each of the tests below. */ -typedef struct test_data_s { - testing t; - int n_threads; /* Number of test threads; constant after init. */ - int loop_count; /* Iteration count for each test thread; constant after init */ - - /* mu_in_use protects i, id, loop_count, and finished_threads. */ - void *mu_in_use; /* points at mu, mutex, or rwmutex depending on which is in use. */ - void (*lock) (void *); /* operations on mu_in_use */ - void (*unlock) (void *); - - nsync_mu mu; - pthread_mutex_t mutex; - pthread_rwlock_t rwmutex; - - int i; /* counter incremented by test loops. */ - volatile int id; /* id of current lock-holding thread in some tests. */ - - nsync_cv done; /* Signalled when finished_threads==n_threads. */ - int finished_threads; /* Count of threads that have finished. */ -} test_data; - -/* Indicate that a thread has finished its operations on test_data - by incrementing td.finished_threads, and signal td.done when it reaches td.n_threads. - See test_data_wait_for_all_threads(). */ -static void test_data_thread_finished (test_data *td) { - (*td->lock) (td->mu_in_use); - td->finished_threads++; - if (td->finished_threads == td->n_threads) { - nsync_cv_broadcast (&td->done); - } - (*td->unlock) (td->mu_in_use); -} - -/* Wait until all td.n_threads have called test_data_thread_finished(), - and then return. */ -static void test_data_wait_for_all_threads (test_data *td) { - (*td->lock) (td->mu_in_use); - while (td->finished_threads != td->n_threads) { - nsync_cv_wait_with_deadline_generic (&td->done, td->mu_in_use, - td->lock, td->unlock, - nsync_time_no_deadline, NULL); - } - (*td->unlock) (td->mu_in_use); -} - -/* --------------------------------------- */ - -/* The body of each thread executed by test_mu_nthread() - and test_mutex_nthread. - *td represents the test data that the threads share, and id is an integer - unique to each test thread. */ -static void counting_loop (test_data *td, int id) { - int n = td->loop_count; - int i = 0; - for (i = 0; i != n; i++) { - (*td->lock) (td->mu_in_use); - td->id = id; - td->i++; - if (td->id != id) { - testing_panic ("td->id != id"); - } - (*td->unlock) (td->mu_in_use); - } - test_data_thread_finished (td); -} - -CLOSURE_DECL_BODY2 (counting, test_data *, int) - -/* Versions of nsync_mu_lock() and nsync_mu_unlock() that take "void *" - arguments, to avoid call through a function pointer of a different type, - which is undefined. */ -static void void_mu_lock (void *mu) { - nsync_mu_lock ((nsync_mu *) mu); -} -static void void_mu_unlock (void *mu) { - nsync_mu_unlock((nsync_mu *) mu); -} - -/* Create a few threads, each of which increments an - integer a fixed number of times, using an nsync_mu for mutual exclusion. - It checks that the integer is incremented the correct number of times. */ -static void test_mu_nthread (testing t) { - int loop_count = 100000; - nsync_time deadline; - deadline = nsync_time_add (nsync_time_now (), nsync_time_ms (1500)); - do { - int i; - test_data td; - memset ((void *) &td, 0, sizeof (td)); - td.t = t; - td.n_threads = 5; - td.loop_count = loop_count; - td.mu_in_use = &td.mu; - td.lock = &void_mu_lock; - td.unlock = &void_mu_unlock; - for (i = 0; i != td.n_threads; i++) { - closure_fork (closure_counting (&counting_loop, &td, i)); - } - test_data_wait_for_all_threads (&td); - if (td.i != td.n_threads*td.loop_count) { - TEST_FATAL (t, ("test_mu_nthread final count inconsistent: want %d, got %d", - td.n_threads*td.loop_count, td.i)); - } - loop_count *= 2; - } while (nsync_time_cmp (nsync_time_now (), deadline) < 0); -} - -/* void pthread_mutex_lock */ -static void void_pthread_mutex_lock (void *mu) { - pthread_mutex_lock ((pthread_mutex_t *) mu); -} - -/* void pthread_mutex_unlock */ -static void void_pthread_mutex_unlock (void *mu) { - pthread_mutex_unlock ((pthread_mutex_t *) mu); -} - -/* Create a few threads, each of which increments an - integer a fixed number of times, using a pthread_mutex_t for mutual exclusion. - It checks that the integer is incremented the correct number of times. */ -static void test_mutex_nthread (testing t) { - int loop_count = 100000; - nsync_time deadline; - deadline = nsync_time_add (nsync_time_now (), nsync_time_ms (1500)); - do { - int i; - test_data td; - memset ((void *) &td, 0, sizeof (td)); - td.t = t; - td.n_threads = 5; - td.loop_count = loop_count; - td.mu_in_use = &td.mutex; - td.lock = &void_pthread_mutex_lock; - td.unlock = &void_pthread_mutex_unlock; - pthread_mutex_init (&td.mutex, NULL); - for (i = 0; i != td.n_threads; i++) { - closure_fork (closure_counting (&counting_loop, &td, i)); - } - test_data_wait_for_all_threads (&td); - if (td.i != td.n_threads*td.loop_count) { - TEST_FATAL (t, ("test_mutex_nthread final count inconsistent: want %d, got %d", - td.n_threads*td.loop_count, td.i)); - } - pthread_mutex_destroy (&td.mutex); - loop_count *= 2; - } while (nsync_time_cmp (nsync_time_now (), deadline) < 0); -} - -/* void pthread_rwlock_wrlock */ -static void void_pthread_rwlock_wrlock (void *mu) { - pthread_rwlock_wrlock ((pthread_rwlock_t *) mu); -} - -/* void pthread_rwlock_unlock */ -static void void_pthread_rwlock_unlock (void *mu) { - pthread_rwlock_unlock ((pthread_rwlock_t *) mu); -} - -/* Create a few threads, each of which increments an - integer a fixed number of times, using a pthread_rwlock_t for mutual exclusion. - It checks that the integer is incremented the correct number of times. */ -static void test_rwmutex_nthread (testing t) { - int loop_count = 100000; - nsync_time deadline; - deadline = nsync_time_add (nsync_time_now (), nsync_time_ms (1500)); - do { - int i; - test_data td; - memset ((void *) &td, 0, sizeof (td)); - td.t = t; - td.n_threads = 5; - td.loop_count = loop_count; - td.mu_in_use = &td.rwmutex; - td.lock = &void_pthread_rwlock_wrlock; - td.unlock = &void_pthread_rwlock_unlock; - pthread_rwlock_init (&td.rwmutex, NULL); - for (i = 0; i != td.n_threads; i++) { - closure_fork (closure_counting (&counting_loop, &td, i)); - } - test_data_wait_for_all_threads (&td); - if (td.i != td.n_threads*td.loop_count) { - TEST_FATAL (t, ("test_mutex_nthread final count inconsistent: want %d, got %d", - td.n_threads*td.loop_count, td.i)); - } - pthread_rwlock_destroy (&td.rwmutex); - loop_count *= 2; - } while (nsync_time_cmp (nsync_time_now (), deadline) < 0); -} - -/* --------------------------------------- */ - -/* The body of each thread executed by test_try_mu_nthread(). - *td represents the test data that the threads share, and id is an integer - unique to each test thread. */ -static void counting_loop_try_mu (test_data *td, int id) { - int i; - int n = td->loop_count; - for (i = 0; i != n; i++) { - while (!nsync_mu_trylock (&td->mu)) { - sched_yield (); - } - td->id = id; - td->i++; - if (td->id != id) { - testing_panic ("td->id != id"); - } - n = td->loop_count; - nsync_mu_unlock (&td->mu); - } - test_data_thread_finished (td); -} - -/* Test that acquiring an nsync_mu with nsync_mu_trylock() - using several threads provides mutual exclusion. */ -static void test_try_mu_nthread (testing t) { - int loop_count = 100000; - nsync_time deadline; - deadline = nsync_time_add (nsync_time_now (), nsync_time_ms (1500)); - do { - int i; - test_data td; - memset ((void *) &td, 0, sizeof (td)); - td.t = t; - td.n_threads = 5; - td.loop_count = loop_count; - td.mu_in_use = &td.mu; - td.lock = &void_mu_lock; - td.unlock = &void_mu_unlock; - for (i = 0; i != td.n_threads; i++) { - closure_fork (closure_counting (&counting_loop_try_mu, &td, i)); - } - test_data_wait_for_all_threads (&td); - if (td.i != td.n_threads*td.loop_count) { - TEST_FATAL (t, ("test_try_mu_nthread final count inconsistent: want %d, got %d", - td.n_threads*td.loop_count, td.i)); - } - loop_count *= 2; - } while (nsync_time_cmp (nsync_time_now (), deadline) < 0); -} - -/* --------------------------------------- */ - -/* An integer protected by a mutex, and with an associated - condition variable that is signalled when the counter reaches 0. */ -typedef struct counter_s { - nsync_mu mu; /* protects value */ - int value; - nsync_cv cv; /* signalled when value becomes 0 */ -} counter; - -/* Return a counter with initial value "initial". */ -static counter *counter_new (int initial) { - counter *c = (counter *) malloc (sizeof (*c)); - memset ((void *) c, 0, sizeof (*c)); - c->value = initial; - return (c); -} - -/* Increment *c by "increment". */ -static void counter_inc (counter *c, int increment) { - if (increment != 0) { - nsync_mu_lock (&c->mu); - c->value += increment; - if (c->value == 0) { - nsync_cv_broadcast (&c->cv); - } - nsync_mu_unlock (&c->mu); - } -} - -/* Wait on *c's condition variable until the counter - becomes 0, or abs_deadline is reached. */ -static int counter_wait_for_zero_with_deadline (counter *c, nsync_time abs_deadline) { - int value; - nsync_mu_rlock (&c->mu); - while (c->value != 0 && - nsync_cv_wait_with_deadline (&c->cv, &c->mu, abs_deadline, NULL) == 0) { - } - value = c->value; - nsync_mu_runlock (&c->mu); - return (value); -} - -/* Wait on *c's condition variable until the counter becomes 0. */ -static void counter_wait_for_zero (counter *c) { - int value = counter_wait_for_zero_with_deadline (c, nsync_time_no_deadline); - if (value != 0) { - testing_panic (smprintf ("wait_for_zero() about to return with " - "non-zero value %d", value)); - } -} - -/* Return the current value of *c. */ -static int counter_value (counter *c) { - int value; - nsync_mu_rlock (&c->mu); - value = c->value; - nsync_mu_runlock (&c->mu); - return (value); -} - -/* --------------------------------------- */ - -CLOSURE_DECL_BODY9 (attempt_trylock, testing , const char *, int, nsync_mu *, - int, int, int *, int, counter *) - -/* Call nsync_mu_trylock(), and compares the result to expected_acquire. - If the lock was acquired, then: - - if expected_value != -1, compare *value against expected_value. - - increment *value. - - if release is non-zero, release the lock before returning. - In any case, the counter *done is decremented. */ -static void attempt_trylock (testing t, const char *id, int verbose, - nsync_mu *mu, int expected_acquire, int release, - int *value, int expected_value, counter *done) { - int acquired = nsync_mu_trylock (mu); - if (acquired != expected_acquire) { - testing_panic (smprintf ("attempt_trylock %s: expected " - "nsync_mu_trylock() to return %d but got %d", - id, expected_acquire, acquired)); - } - if (verbose) { - TEST_LOG (t, ("attempt_trylock %s %d\n", id, acquired)); - } - if (acquired) { - nsync_mu_assert_held (mu); - if (expected_value != -1 && *value != expected_value) { - testing_panic (smprintf ("attempt_trylock %s expected " - "value %d, *value=%d", - id, expected_value, *value)); - } - (*value)++; - if (verbose) { - TEST_LOG (t, ("attempt_trylock %s incremented value to %d\n", id, *value)); - } - if (release) { - nsync_mu_unlock (mu); - } - } - counter_inc (done, -1); -} - -/* Call nsync_mu_rtrylock(), and compare the result to expected_acquire. - If the lock was acquired, then: - - if expected_value != -1, compare *value against expected_value. - - if release is non-zero, release the lock before returning. - In any case, decrement *done. */ -static void attempt_rtrylock (testing t, const char *id, int verbose, - nsync_mu *mu, int expected_acquire, int release, - int *value, int expected_value, counter *done) { - int acquired = nsync_mu_rtrylock (mu); - if (acquired != expected_acquire) { - testing_panic (smprintf ("attempt_rtrylock %s: expected " - "nsync_mu_rtrylock() to return %d but got %d", - id, expected_acquire, acquired)); - } - if (verbose) { - TEST_LOG (t, ("attempt_rtrylock %s %d\n", id, acquired)); - } - if (acquired) { - nsync_mu_rassert_held (mu); - if (expected_value != -1 && *value != expected_value) { - testing_panic (smprintf ("attempt_rtrylock %s expected " - "value %d, *value=%d", - id, expected_value, *value)); - } - if (release) { - nsync_mu_runlock (mu); - } - } - counter_inc (done, -1); -} - -CLOSURE_DECL_BODY9 (lock_unlock, testing, const char *, int, nsync_mu *, - int *, int, nsync_time, counter *, counter *) - -/* First acquire *mu, then: - - if expected_value != -1, compare *value against expected_value. - - increment *value. - - sleep for "sleep". - Then release *mu and decrement *done. */ -static void lock_unlock (testing t, const char *id, int verbose, nsync_mu *mu, int *value, - int expected_value, nsync_time sleep, counter *sleeping, counter *done) { - if (verbose) { - TEST_LOG (t, ("lock_unlock %s\n", id)); - } - if (sleeping != NULL) { - counter_inc (sleeping, -1); - } - nsync_mu_lock (mu); - nsync_mu_assert_held (mu); - if (expected_value != -1 && *value != expected_value) { - testing_panic (smprintf ("lock_unlock %s expected " - "value %d, *value=%d", - id, expected_value, *value)); - } - (*value)++; - if (verbose) { - TEST_LOG (t, ("lock_unlock %s incremented value to %d\n", id, *value)); - } - nsync_time_sleep (sleep); - nsync_mu_unlock (mu); - counter_inc (done, -1); -} - -/* First acquire *mu in read mode, then: - - if expected_value != -1, compare *value against expected_value. - - sleep for "sleep". - Then release *mu and decrement *done. */ -static void rlock_runlock (testing t, const char *id, int verbose, nsync_mu *mu, - int *value, int expected_value, nsync_time sleep, - counter *sleeping, counter *done) { - if (verbose) { - TEST_LOG (t, ("rlock_runlock %s\n", id)); - } - if (sleeping != NULL) { - counter_inc (sleeping, -1); - } - nsync_mu_rlock (mu); - nsync_mu_rassert_held (mu); - if (expected_value != -1 && *value != expected_value) { - testing_panic (smprintf ("rlock_runlock %s expected " - "value %d, *value=%d", id, expected_value, *value)); - } - nsync_time_sleep (sleep); - nsync_mu_runlock (mu); - counter_inc (done, -1); -} - -/* Check that the time since start_time is between expected_duration-1ms. - If the time exceeds expected_duration+slop_duration, return 1, else 0. */ -static int check_times (testing t, const char *id, nsync_time start_time, - nsync_time expected_duration, nsync_time slop_duration) { - int exceeds_count = 0; - nsync_time now; - nsync_time measured_duration; - now = nsync_time_now (); - measured_duration = nsync_time_sub (now, start_time); - if (nsync_time_cmp (measured_duration, - nsync_time_sub (expected_duration, nsync_time_ms (5))) < 0) { - char *m_str = nsync_time_str (measured_duration, 2); - char *e_str = nsync_time_str (expected_duration, 2); - TEST_ERROR (t, ("check_times %s too short a delay: %s instead of %s", - id, m_str, e_str)); - free (m_str); - free (e_str); - } - if (nsync_time_cmp (nsync_time_add (expected_duration, slop_duration), measured_duration) < 0) { - exceeds_count++; - } - return (exceeds_count); -} - -/* Check the operation of nsync_mu as a reader/writer lock. */ -static void test_rlock (testing t) { - int loop; - int i; - int max_write_wait_exceeded; - int max_read_wait_exceeded; - nsync_time time_unit; - nsync_time slop_duration; - nsync_time delay_duration; - nsync_time writer_duration; - nsync_time reader_duration; - static const int loop_count = 5; - static const int read_threads = 3; - static const int limit = 3; - static const int verbose = 0; - max_write_wait_exceeded = 0; - max_read_wait_exceeded = 0; - - time_unit = nsync_time_ms (100); - slop_duration = nsync_time_add (nsync_time_add (time_unit, time_unit), time_unit); - delay_duration = time_unit; - writer_duration = time_unit; - reader_duration = nsync_time_add (time_unit, time_unit); - - max_write_wait_exceeded = 0; - max_read_wait_exceeded = 0; - for (loop = 0; loop != loop_count; loop++) { - counter *lock_unlock_sleeping; - counter *rlock_runlock_sleeping; - counter *lock_unlock_done; - counter *rlock_runlock_done; - nsync_time read_start_time; - nsync_mu mu; - int value = 0; - counter *thread_done; - - nsync_time start_time; - nsync_mu_init (&mu); - start_time = nsync_time_now (); - - /* ------------------------------------ */ - /* Acquire lock with nsync_mu_rtrylock(). This thread will - hold a read lock until the next line with =====. */ - thread_done = counter_new (1); - attempt_rtrylock (t, "a", verbose, &mu, 1, 0, &value, 0, thread_done); - counter_wait_for_zero (thread_done); - - nsync_mu_rassert_held (&mu); - - counter_inc (thread_done, 1); - /* Can get read lock holding read lock. */ - closure_fork (closure_attempt_trylock (&attempt_rtrylock, - t, "b", verbose, &mu, 1, 1, &value, 0, thread_done)); - counter_wait_for_zero (thread_done); - - nsync_mu_rassert_held (&mu); - - counter_inc (thread_done, 1); - /* Can't get write lock holding read lock. */ - closure_fork (closure_attempt_trylock (&attempt_trylock, t, "c", verbose, - &mu, 0, 1, &value, -1, thread_done)); - counter_wait_for_zero (thread_done); - - if (!nsync_mu_is_reader (&mu)) { - TEST_FATAL(t, ("expected mu held in reader mode")); - } - - counter_inc (thread_done, 1); - closure_fork (closure_lock_unlock (&rlock_runlock, t, "d", verbose, - &mu, &value, 0, nsync_time_zero /*no delay*/, - NULL, thread_done)); - counter_wait_for_zero (thread_done); - - nsync_mu_rassert_held (&mu); - - lock_unlock_done = counter_new (1); - lock_unlock_sleeping = counter_new (1); - closure_fork (closure_lock_unlock (&lock_unlock, t, "e", verbose, - &mu, &value, 0, writer_duration, - lock_unlock_sleeping, lock_unlock_done)); - - counter_wait_for_zero (lock_unlock_sleeping); - nsync_time_sleep (delay_duration); /* give time for lock_unlock() thread to wait. */ - - nsync_mu_rassert_held (&mu); - - rlock_runlock_done = counter_new (read_threads); - rlock_runlock_sleeping = counter_new (read_threads); - for (i = 0; i != read_threads; i++) { - /* read lock will be acquired after lock_unlock() completes */ - closure_fork (closure_lock_unlock (&rlock_runlock, t, "f", verbose, - &mu, &value, 1, reader_duration, - rlock_runlock_sleeping, - rlock_runlock_done)); - } - - nsync_mu_rassert_held (&mu); - - counter_wait_for_zero (rlock_runlock_sleeping); - nsync_time_sleep (delay_duration); /* time for rlock_runlock() threads to wait. */ - - nsync_mu_rassert_held (&mu); - - if (counter_value (lock_unlock_done) == 0) { - TEST_FATAL (t, ("thread was able to acquire write lock while read lock held")); - } - if (counter_value (rlock_runlock_done) == 0) { - TEST_FATAL (t, ("thread was able to acquire read lock with " - "other reader and waiting writer")); - } - - nsync_mu_rassert_held (&mu); - - counter_inc (thread_done, 1); - /* Still can't get write lock. */ - closure_fork (closure_attempt_trylock (&attempt_trylock, t, "g", verbose, - &mu, 0, 1, &value, -1, thread_done)); - counter_wait_for_zero (thread_done); - - counter_inc (thread_done, 1); - /* Now can't get read lock because a writer is waiting. */ - closure_fork (closure_attempt_trylock (&attempt_rtrylock, t, "h", verbose, - &mu, 0, 1, &value, -1, thread_done)); - counter_wait_for_zero (thread_done); - - nsync_mu_runlock (&mu); - /* ==================================== */ - - read_start_time = nsync_time_now (); - counter_wait_for_zero (lock_unlock_done); /* Now can get write lock. */ - max_write_wait_exceeded += check_times (t, "i", start_time, - nsync_time_add (nsync_time_add (delay_duration, delay_duration), writer_duration), - slop_duration); - - counter_wait_for_zero (rlock_runlock_done); /* And now an get read lock again. */ - max_read_wait_exceeded += check_times (t, "j", read_start_time, - reader_duration, slop_duration); - - free (thread_done); - free (lock_unlock_done); - free (rlock_runlock_done); - free (lock_unlock_sleeping); - free (rlock_runlock_sleeping); - } - if (verbose) { - TEST_LOG (t, ("read lock max_write_wait_exceeded %d max_read_wait_exceeded %d\n", - max_write_wait_exceeded, max_read_wait_exceeded)); - } - if (max_write_wait_exceeded > limit) { - TEST_ERROR (t, ("lock_unlock() took too long %d " - "(more than %d) times out of %d", - max_write_wait_exceeded, limit, loop_count)); - } - if (max_read_wait_exceeded > limit) { - TEST_ERROR (t, ("rlock_runlock() took too long %d " - "(more than %d) times out of %d", - max_read_wait_exceeded, limit, loop_count)); - } - - max_write_wait_exceeded = 0; - max_read_wait_exceeded = 0; - for (loop = 0; loop != loop_count; loop++) { - counter *lock_unlock_sleeping; - counter *rlock_runlock_sleeping; - counter *lock_unlock_done; - counter *rlock_runlock_done; - nsync_time read_start_time; - nsync_mu mu; - int value = 0; - counter *thread_done; - - nsync_time start_time; - - nsync_mu_init (&mu); - start_time = nsync_time_now (); - - /* ------------------------------------ */ - /* Acquire lock with nsync_mu_trylock(). This thread will hold - a write lock until the next line with =====. */ - thread_done = counter_new (1); - attempt_trylock (t, "A", verbose, &mu, 1, 0, &value, 0, thread_done); - counter_wait_for_zero (thread_done); - - nsync_mu_assert_held (&mu); - nsync_mu_rassert_held (&mu); - - counter_inc (thread_done, 1); - /* Can't get read lock while holding write lock. */ - closure_fork (closure_attempt_trylock (&attempt_rtrylock, t, "B", verbose, - &mu, 0, 1, &value, -1, thread_done)); - counter_wait_for_zero (thread_done); - - if (nsync_mu_is_reader (&mu)) { - TEST_FATAL (t, ("expected mu held in write mode")); - } - nsync_mu_assert_held (&mu); - nsync_mu_rassert_held (&mu); - - counter_inc (thread_done, 1); - /* Can't get write lock while holding write lock. */ - closure_fork (closure_attempt_trylock (&attempt_trylock, t, "C", verbose, - &mu, 0, 1, &value, -1, thread_done)); - counter_wait_for_zero (thread_done); - - nsync_mu_assert_held (&mu); - nsync_mu_rassert_held (&mu); - - lock_unlock_done = counter_new (1); - lock_unlock_sleeping = counter_new (1); - closure_fork (closure_lock_unlock (&lock_unlock, t, "D", verbose, - &mu, &value, 1, writer_duration, - lock_unlock_sleeping, lock_unlock_done)); - - counter_wait_for_zero (lock_unlock_sleeping); - nsync_time_sleep (delay_duration); /* give time for lock_unlock() thread to wait. */ - - nsync_mu_assert_held (&mu); - nsync_mu_rassert_held (&mu); - - rlock_runlock_done = counter_new (read_threads); - rlock_runlock_sleeping = counter_new (read_threads); - for (i = 0; i != read_threads; i++) { - /* not guaranteed will complete after lock_unlock() above */ - closure_fork (closure_lock_unlock (&rlock_runlock, t, "E", verbose, - &mu, &value, -1, reader_duration, - rlock_runlock_sleeping, - rlock_runlock_done)); - } - - nsync_mu_assert_held (&mu); - nsync_mu_rassert_held (&mu); - - counter_wait_for_zero (rlock_runlock_sleeping); - nsync_time_sleep (delay_duration); /* time for rlock_runlock() threads to wait. */ - - nsync_mu_assert_held (&mu); - nsync_mu_rassert_held (&mu); - - if (counter_value (lock_unlock_done) == 0) { - TEST_FATAL (t, ("thread was able to acquire write lock " - "while other write lock held")); - } - if (counter_value (rlock_runlock_done) == 0) { - TEST_FATAL (t, ("thread was able to acquire read lock " - "while write lock held")); - } - - nsync_mu_assert_held (&mu); - nsync_mu_rassert_held (&mu); - - counter_inc (thread_done, 1); - /* Still can't get read lock while holding write lock. */ - closure_fork (closure_attempt_trylock (&attempt_rtrylock, t, "F", verbose, - &mu, 0, 1, &value, -1, thread_done)); - counter_wait_for_zero (thread_done); - - nsync_mu_assert_held (&mu); - nsync_mu_rassert_held (&mu); - - counter_inc (thread_done, 1); - /* Still can't get write lock while holding write lock. */ - closure_fork (closure_attempt_trylock (&attempt_trylock, t, "G", verbose, - &mu, 0, 1, &value, -1, thread_done)); - counter_wait_for_zero (thread_done); - - nsync_mu_assert_held (&mu); - nsync_mu_rassert_held (&mu); - - nsync_mu_unlock (&mu); - /* ==================================== */ - - read_start_time = nsync_time_now (); - counter_wait_for_zero (lock_unlock_done); /* Now can get write lock. */ - max_write_wait_exceeded += check_times (t, "H", start_time, - nsync_time_add (nsync_time_add (delay_duration, delay_duration), writer_duration), - slop_duration); - - counter_wait_for_zero (rlock_runlock_done); /* And now can get read lock again. */ - max_read_wait_exceeded += check_times (t, "I", read_start_time, - reader_duration, slop_duration); - - free (thread_done); - free (lock_unlock_done); - free (rlock_runlock_done); - free (lock_unlock_sleeping); - free (rlock_runlock_sleeping); - } - if (verbose) { - TEST_LOG (t, ("write lock max_write_wait_exceeded %d " - "max_read_wait_exceeded %d\n", - max_write_wait_exceeded, max_read_wait_exceeded)); - } - if (max_write_wait_exceeded > limit) { - TEST_ERROR (t, ("lock_unlock() took too long %d (more than %d) " - "times out of %d", - max_write_wait_exceeded, limit, loop_count)); - } - if (max_read_wait_exceeded > limit) { - TEST_ERROR (t, ("rlock_runlock() took too long %d (more than %d) " - "times out of %d", - max_read_wait_exceeded, limit, loop_count)); - } -} - -/* --------------------------------------- */ - -/* Measure the performance of an uncontended nsync_mu. */ -static void benchmark_mu_uncontended (testing t) { - int i; - int n = testing_n (t); - nsync_mu mu; - nsync_mu_init (&mu); - for (i = 0; i != n; i++) { - nsync_mu_lock (&mu); - nsync_mu_unlock (&mu); - } -} - -/* Return whether int *value is one. */ -static int int_is_1 (const void *value) { return (*(const int *)value == 1); } - -/* Return whether int *value is two. */ -static int int_is_2 (const void *value) { return (*(const int *)value == 2); } - -/* Return whether int *value is three. */ -static int int_is_3 (const void *value) { return (*(const int *)value == 3); } - -/* Set *value to 1, wait for it to become 2, then set it to 3. *value is under - *mu */ -static void waiter (nsync_mu *mu, int *value) { - nsync_mu_lock (mu); - *value = 1; - nsync_mu_wait (mu, &int_is_2, value, NULL); - *value = 3; - nsync_mu_unlock (mu); -} - -CLOSURE_DECL_BODY2 (waiter, nsync_mu *, int *) - -/* Measure the performance of an uncontended nsync_mu - with a blocked waiter. */ -static void benchmark_mu_uncontended_waiter (testing t) { - int i; - int n = testing_n (t); - nsync_mu mu; - int value = 0; - nsync_mu_init (&mu); - closure_fork (closure_waiter (&waiter, &mu, &value)); - nsync_mu_lock (&mu); - nsync_mu_wait (&mu, &int_is_1, &value, NULL); - nsync_mu_unlock (&mu); - for (i = 0; i != n; i++) { - nsync_mu_lock (&mu); - nsync_mu_unlock (&mu); - } - nsync_mu_lock (&mu); - value = 2; - nsync_mu_wait (&mu, &int_is_3, &value, NULL); - nsync_mu_unlock (&mu); -} - -/* Measure the performance of an uncontended nsync_mu - with a blocked waiter using nsync_mu_unlock_without_wakeup. */ -static void benchmark_mu_uncontended_no_wakeup (testing t) { - int i; - int n = testing_n (t); - nsync_mu mu; - int value = 0; - nsync_mu_init (&mu); - closure_fork (closure_waiter (&waiter, &mu, &value)); - nsync_mu_lock (&mu); - nsync_mu_wait (&mu, &int_is_1, &value, NULL); - nsync_mu_unlock (&mu); - for (i = 0; i != n; i++) { - nsync_mu_lock (&mu); - nsync_mu_unlock_without_wakeup (&mu); - } - nsync_mu_lock (&mu); - value = 2; - nsync_mu_wait (&mu, &int_is_3, &value, NULL); - nsync_mu_unlock (&mu); -} - -/* Measure the performance of an uncontended - nsync_mu in read mode. */ -static void benchmark_rmu_uncontended (testing t) { - int i; - int n = testing_n (t); - nsync_mu mu; - nsync_mu_init (&mu); - for (i = 0; i != n; i++) { - nsync_mu_rlock (&mu); - nsync_mu_runlock (&mu); - } -} - -/* Measure the performance of an uncontended nsync_mu - in read mode with a blocked waiter. */ -static void benchmark_rmu_uncontended_waiter (testing t) { - int i; - int n = testing_n (t); - nsync_mu mu; - int value = 0; - nsync_mu_init (&mu); - closure_fork (closure_waiter (&waiter, &mu, &value)); - nsync_mu_lock (&mu); - nsync_mu_wait (&mu, &int_is_1, &value, NULL); - nsync_mu_unlock (&mu); - for (i = 0; i != n; i++) { - nsync_mu_rlock (&mu); - nsync_mu_runlock (&mu); - } - nsync_mu_lock (&mu); - value = 2; - nsync_mu_wait (&mu, &int_is_3, &value, NULL); - nsync_mu_unlock (&mu); -} - -/* Measure the performance of an uncontended pthread_mutex_t. */ -static void benchmark_mutex_uncontended (testing t) { - int i; - int n = testing_n (t); - pthread_mutex_t mu; - pthread_mutex_init (&mu, NULL); - for (i = 0; i != n; i++) { - pthread_mutex_lock (&mu); - pthread_mutex_unlock (&mu); - } - pthread_mutex_destroy (&mu); -} - -/* Measure the performance of an uncontended pthread_rwlock_t. */ -static void benchmark_wmutex_uncontended (testing t) { - int i; - int n = testing_n (t); - pthread_rwlock_t mu; - pthread_rwlock_init (&mu, NULL); - for (i = 0; i != n; i++) { - pthread_rwlock_wrlock (&mu); - pthread_rwlock_unlock (&mu); - } - pthread_rwlock_destroy (&mu); -} - -/* Measure the performance of an uncontended - pthread_rwlock_t in read mode. */ -static void benchmark_rmutex_uncontended (testing t) { - int i; - int n = testing_n (t); - pthread_rwlock_t mu; - pthread_rwlock_init (&mu, NULL); - for (i = 0; i != n; i++) { - pthread_rwlock_rdlock (&mu); - pthread_rwlock_unlock (&mu); - } - pthread_rwlock_destroy (&mu); -} - -/* --------------------------------------- - Benchmarks for contended locks. */ - -/* It's hard to write these as benchmark functions, since we wish to measure - throughput over an extended period (a second or two), rather than get the - latency of a few iterations. */ - -/* A contended_state represents state shared between threads - in the contended benchmarks. */ -typedef struct contended_state_s { - testing t; - - /* locks to test */ - nsync_mu mu; - pthread_mutex_t mutex; - pthread_rwlock_t rwmutex; - int count; /* counter protected by a lock above */ - - nsync_mu start_done_mu; - int start; /* whether threads should start, under start_done_mu */ - int not_yet_done; /* threads not yet complete, under start_done_mu */ -} contended_state; - -static int contended_state_may_start (const void *v) { - return (((const contended_state *)v)->start); -} - -static int contended_state_all_done (const void *v) { - return (((const contended_state *)v)->not_yet_done == 0); -} - -/* Wait for cs.start to become non-zero, then loop, acquiring and - releasing mu on each iteration until cs.deadline is reached, then decrement - cs.not_yet_done. */ -static void contended_state_contend_loop (contended_state *cs, - void *mu, void (*lock) (void *), - void (*unlock) (void *)) { - int n = testing_n (cs->t); - int j; - int i; - nsync_mu_rlock (&cs->start_done_mu); - nsync_mu_wait (&cs->start_done_mu, &contended_state_may_start, cs, NULL); - nsync_mu_runlock (&cs->start_done_mu); - - for (j = 0; j < n; j += 10000) { - for (i = 0; i != 10000; i++) { - (*lock) (mu); - cs->count++; - (*unlock) (mu); - } - } - - nsync_mu_lock (&cs->start_done_mu); - cs->not_yet_done--; - nsync_mu_unlock (&cs->start_done_mu); -} - -typedef void (*func_any) (void *); -CLOSURE_DECL_BODY4 (contended_state_contend_loop, contended_state *, void *, func_any, func_any) - -/* Start the threads in a contended test, wait for them to finish, - and print the number of iterations achieved. */ -static void contended_state_run_test (contended_state *cs, testing t, - void *mu, void (*lock) (void *), - void (*unlock) (void *)) { - int i; - cs->t = t; - cs->not_yet_done = 4; /* number of threads */ - cs->start = 0; - cs->count = 0; - for (i = 0; i != cs->not_yet_done; i++) { - closure_fork (closure_contended_state_contend_loop ( - &contended_state_contend_loop, cs, mu, lock, unlock)); - } - nsync_mu_lock (&cs->start_done_mu); - cs->start = 1; - nsync_mu_wait (&cs->start_done_mu, &contended_state_all_done, cs, NULL); - nsync_mu_unlock (&cs->start_done_mu); -} - -/* Measure the performance of highly contended - nsync_mu locks, with small critical sections. */ -static void benchmark_mu_contended (testing t) { - contended_state cs; - memset ((void *) &cs, 0, sizeof (cs)); - contended_state_run_test (&cs, t, &cs.mu, (void (*) (void*))&nsync_mu_lock, - (void (*) (void*))&nsync_mu_unlock); -} - -/* Measure the performance of highly contended - pthread_mutex_t locks, with small critical sections. */ -static void benchmark_mutex_contended (testing t) { - contended_state cs; - memset ((void *) &cs, 0, sizeof (cs)); - pthread_mutex_init (&cs.mutex, NULL); - contended_state_run_test (&cs, t, &cs.mutex, &void_pthread_mutex_lock, - &void_pthread_mutex_unlock); - pthread_mutex_destroy (&cs.mutex); -} - -/* Measure the performance of highly contended - pthread_rwlock_t locks, with small critical sections. */ -static void benchmark_wmutex_contended (testing t) { - contended_state cs; - memset ((void *) &cs, 0, sizeof (cs)); - pthread_rwlock_init (&cs.rwmutex, NULL); - contended_state_run_test (&cs, t, &cs.rwmutex, &void_pthread_rwlock_wrlock, - &void_pthread_rwlock_unlock); - pthread_rwlock_destroy (&cs.rwmutex); -} - -int main (int argc, char *argv[]) { - testing_base tb = testing_new (argc, argv, 0); - - TEST_RUN (tb, test_rlock); - TEST_RUN (tb, test_mu_nthread); - TEST_RUN (tb, test_mutex_nthread); - TEST_RUN (tb, test_rwmutex_nthread); - TEST_RUN (tb, test_try_mu_nthread); - - BENCHMARK_RUN (tb, benchmark_mu_contended); - BENCHMARK_RUN (tb, benchmark_mutex_contended); - BENCHMARK_RUN (tb, benchmark_wmutex_contended); - - BENCHMARK_RUN (tb, benchmark_mu_uncontended); - BENCHMARK_RUN (tb, benchmark_rmu_uncontended); - BENCHMARK_RUN (tb, benchmark_mutex_uncontended); - BENCHMARK_RUN (tb, benchmark_wmutex_uncontended); - BENCHMARK_RUN (tb, benchmark_rmutex_uncontended); - BENCHMARK_RUN (tb, benchmark_mu_uncontended_waiter); - BENCHMARK_RUN (tb, benchmark_mu_uncontended_no_wakeup); - BENCHMARK_RUN (tb, benchmark_rmu_uncontended_waiter); - - return (testing_base_exit (tb)); -} diff --git a/examples/thread.c b/examples/thread.c index d59a0eb4a..f019d7107 100644 --- a/examples/thread.c +++ b/examples/thread.c @@ -7,7 +7,7 @@ │ • http://creativecommons.org/publicdomain/zero/1.0/ │ ╚─────────────────────────────────────────────────────────────────*/ #endif -#include "libc/thread/thread.h" +#include "libc/intrin/pthread.h" #include "libc/stdio/stdio.h" /** diff --git a/examples/vga.c b/examples/vga.c deleted file mode 100644 index 262dc2a30..000000000 --- a/examples/vga.c +++ /dev/null @@ -1,60 +0,0 @@ -#if 0 -/*─────────────────────────────────────────────────────────────────╗ -│ To the extent possible under law, Justine Tunney has waived │ -│ all copyright and related or neighboring rights to this file, │ -│ as it is written in the following disclaimers: │ -│ • http://unlicense.org/ │ -│ • http://creativecommons.org/publicdomain/zero/1.0/ │ -╚─────────────────────────────────────────────────────────────────*/ -#endif -#include "libc/calls/calls.h" -#include "libc/calls/termios.h" -#include "libc/isystem/unistd.h" -#include "libc/math.h" -#include "libc/stdio/stdio.h" -#include "libc/str/str.h" -#include "libc/sysv/consts/termios.h" - -/** - * @fileoverview Bare Metal VGA TTY demo. - * - * This program can boot as an operating system. Try it out: - * - * make -j8 o//examples/vga.com - * qemu-system-x86_64 -hda o//examples/vga.com -serial stdio - * - * Please note that, by default, APE binaries only use the serial port - * for stdio. To get the VGA console as an added bonus: - * - * STATIC_YOINK("vga_console"); - * - * Should be added to the top of your main() program source file. - */ - -STATIC_YOINK("vga_console"); - -int main(int argc, char *argv[]) { - volatile long double x = -.5; - volatile long double y = 1.5; - struct termios tio; - char buf[16]; - ssize_t res; - if (tcgetattr(0, &tio) != -1) { - tio.c_lflag &= ~(ECHO | ICANON); - tcsetattr(0, TCSANOW, &tio); - } - write(1, "\e[5n", 4); - res = read(0, buf, 4); - if (res != 4 || memcmp(buf, "\e[0n", 4) != 0) { - printf("res = %d?\n", res); - return -1; - } - printf("Hello World! %.19Lg\n", atan2l(x, y)); - - // read/print loop so machine doesn't reset on metal - for (;;) { - if ((res = readansi(0, buf, 16)) > 0) { - printf("got %`'.*s\r\n", res, buf); - } - } -} diff --git a/examples/walk.c b/examples/walk.c index 6b6253a70..b9eca8eba 100644 --- a/examples/walk.c +++ b/examples/walk.c @@ -7,12 +7,16 @@ │ • http://creativecommons.org/publicdomain/zero/1.0/ │ ╚─────────────────────────────────────────────────────────────────*/ #endif +#include "libc/calls/calls.h" +#include "libc/calls/struct/stat.h" #include "libc/errno.h" +#include "libc/runtime/gc.h" #include "libc/runtime/runtime.h" #include "libc/stdio/stdio.h" #include "libc/str/str.h" #include "libc/sysv/consts/exit.h" #include "libc/sysv/consts/s.h" +#include "libc/x/x.h" #include "third_party/musl/ftw.h" /** diff --git a/libc/assert.h b/libc/assert.h index e6ef79237..ce6933b2a 100644 --- a/libc/assert.h +++ b/libc/assert.h @@ -17,20 +17,6 @@ void __assert_fail(const char *, const char *, int) hidden relegated; #define static_assert _Static_assert #endif -#define _unassert(x) \ - do { \ - if (__builtin_expect(!(x), 0)) { \ - unreachable; \ - } \ - } while (0) - -#define _npassert(x) \ - do { \ - if (__builtin_expect(!(x), 0)) { \ - notpossible; \ - } \ - } while (0) - COSMOPOLITAN_C_END_ #endif /* !(__ASSEMBLER__ + __LINKER__ + 0) */ #endif /* COSMOPOLITAN_LIBC_ASSERT_H_ */ diff --git a/libc/atomic.h b/libc/atomic.h index f132e5ed2..dd8f831f8 100644 --- a/libc/atomic.h +++ b/libc/atomic.h @@ -1,6 +1,5 @@ #ifndef COSMOPOLITAN_LIBC_ATOMIC_H_ #define COSMOPOLITAN_LIBC_ATOMIC_H_ -#include "libc/inttypes.h" #if !(__ASSEMBLER__ + __LINKER__ + 0) COSMOPOLITAN_C_START_ diff --git a/libc/calls/_timespec_real.c b/libc/calls/_timespec_real.c index 80912dc1f..805c4b9d6 100644 --- a/libc/calls/_timespec_real.c +++ b/libc/calls/_timespec_real.c @@ -16,6 +16,7 @@ │ TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR │ │ PERFORMANCE OF THIS SOFTWARE. │ ╚─────────────────────────────────────────────────────────────────────────────*/ +#include "libc/assert.h" #include "libc/calls/struct/timespec.h" #include "libc/sysv/consts/clock.h" #include "libc/sysv/consts/nr.h" @@ -25,6 +26,6 @@ struct timespec _timespec_real(void) { int ax, dx; struct timespec ts; ax = clock_gettime(CLOCK_REALTIME_FAST, &ts); - if (ax) notpossible; + assert(!ax); return ts; } diff --git a/libc/calls/_timespec_sub.c b/libc/calls/_timespec_sub.c index 0f1fe4c35..5ba0d0fb4 100644 --- a/libc/calls/_timespec_sub.c +++ b/libc/calls/_timespec_sub.c @@ -21,12 +21,12 @@ /** * Subtracts two nanosecond timestamps. */ -struct timespec _timespec_sub(struct timespec a, struct timespec b) { - a.tv_sec -= b.tv_sec; - if (a.tv_nsec < b.tv_nsec) { - a.tv_nsec += 1000000000; - a.tv_sec--; +struct timespec _timespec_sub(struct timespec x, struct timespec y) { + x.tv_sec -= y.tv_sec; + x.tv_nsec -= y.tv_nsec; + if (x.tv_nsec < 0) { + x.tv_nsec += 1000000000; + x.tv_sec -= 1; } - a.tv_nsec -= b.tv_nsec; - return a; + return x; } diff --git a/libc/intrin/arememoryintervalsok.c b/libc/calls/arememoryintervalsok.c similarity index 100% rename from libc/intrin/arememoryintervalsok.c rename to libc/calls/arememoryintervalsok.c diff --git a/libc/calls/calls.mk b/libc/calls/calls.mk index 10177d083..bab0d35be 100644 --- a/libc/calls/calls.mk +++ b/libc/calls/calls.mk @@ -79,6 +79,14 @@ o/$(MODE)/libc/calls/vdsofunc.greg.o: private \ -ffreestanding \ -fno-sanitize=address +# we can't use asan because: +# asan guard pages haven't been allocated yet +o/$(MODE)/libc/calls/directmap.o \ +o/$(MODE)/libc/calls/directmap-nt.o: private \ + OVERRIDE_COPTS += \ + -ffreestanding \ + -fno-sanitize=address + # we can't use asan because: # ntspawn allocates 128kb of heap memory via win32 o/$(MODE)/libc/calls/ntspawn.o \ @@ -136,6 +144,12 @@ o/$(MODE)/libc/calls/ioctl-siocgifconf-nt.o: private \ -ffunction-sections \ -fdata-sections +# we want small code size because: +# to keep .text.head under 4096 bytes +o/$(MODE)/libc/calls/mman.greg.o: private \ + OVERRIDE_COPTS += \ + -Os + # we always want -Os because: # va_arg codegen is very bloated in default mode o//libc/calls/open.o \ diff --git a/libc/calls/clock_gettime-mono.c b/libc/calls/clock_gettime-mono.c index 550af8dcd..42a88162e 100644 --- a/libc/calls/clock_gettime-mono.c +++ b/libc/calls/clock_gettime-mono.c @@ -17,7 +17,7 @@ │ PERFORMANCE OF THIS SOFTWARE. │ ╚─────────────────────────────────────────────────────────────────────────────*/ #include "libc/calls/struct/timespec.h" -#include "libc/thread/thread.h" +#include "libc/intrin/pthread.h" #include "libc/nexgen32e/rdtsc.h" #include "libc/nexgen32e/x86feature.h" #include "libc/sysv/consts/clock.h" diff --git a/libc/intrin/directmap-metal.c b/libc/calls/directmap-metal.c similarity index 100% rename from libc/intrin/directmap-metal.c rename to libc/calls/directmap-metal.c diff --git a/libc/intrin/directmap-nt.c b/libc/calls/directmap-nt.c similarity index 100% rename from libc/intrin/directmap-nt.c rename to libc/calls/directmap-nt.c diff --git a/libc/intrin/directmap.c b/libc/calls/directmap.c similarity index 100% rename from libc/intrin/directmap.c rename to libc/calls/directmap.c diff --git a/libc/calls/extend.internal.h b/libc/calls/extend.internal.h deleted file mode 100644 index 6de407ddc..000000000 --- a/libc/calls/extend.internal.h +++ /dev/null @@ -1,10 +0,0 @@ -#ifndef COSMOPOLITAN_LIBC_CALLS_EXTEND_INTERNAL_H_ -#define COSMOPOLITAN_LIBC_CALLS_EXTEND_INTERNAL_H_ -#if !(__ASSEMBLER__ + __LINKER__ + 0) -COSMOPOLITAN_C_START_ - -void *_extend(void *, size_t, void *, intptr_t); - -COSMOPOLITAN_C_END_ -#endif /* !(__ASSEMBLER__ + __LINKER__ + 0) */ -#endif /* COSMOPOLITAN_LIBC_CALLS_EXTEND_INTERNAL_H_ */ diff --git a/libc/calls/fcntl-nt.c b/libc/calls/fcntl-nt.c index 05626e755..9ddd29789 100644 --- a/libc/calls/fcntl-nt.c +++ b/libc/calls/fcntl-nt.c @@ -44,34 +44,22 @@ #include "libc/sysv/consts/o.h" #include "libc/sysv/errfuns.h" -bool __force_sqlite_to_work_until_we_can_fix_it; - static textwindows int sys_fcntl_nt_dupfd(int fd, int cmd, int start) { - if (start < 0) return einval(); return sys_dup_nt(fd, -1, (cmd == F_DUPFD_CLOEXEC ? O_CLOEXEC : 0), start); } static textwindows int sys_fcntl_nt_lock(struct Fd *f, int cmd, uintptr_t arg) { - int e; struct flock *l; uint32_t flags, err; + struct NtOverlapped ov; int64_t pos, off, len, size; struct NtByHandleFileInformation info; - - if (!GetFileInformationByHandle(f->handle, &info)) { - return __winerr(); - } - - pos = 0; - if (!SetFilePointerEx(f->handle, 0, &pos, SEEK_CUR)) { - return __winerr(); - } - + if (!GetFileInformationByHandle(f->handle, &info)) return __winerr(); + if (!SetFilePointerEx(f->handle, 0, &pos, SEEK_CUR)) return __winerr(); l = (struct flock *)arg; len = l->l_len; off = l->l_start; size = (uint64_t)info.nFileSizeHigh << 32 | info.nFileSizeLow; - switch (l->l_whence) { case SEEK_SET: break; @@ -84,64 +72,37 @@ static textwindows int sys_fcntl_nt_lock(struct Fd *f, int cmd, uintptr_t arg) { default: return einval(); } - - if (!len) { - len = size - off; - } - - if (off < 0 || len < 0) { - return einval(); - } - - bool32 ok; - struct NtOverlapped ov = {.hEvent = f->handle, - .Pointer = (void *)(uintptr_t)off}; - + if (!len) len = size - off; + if (off < 0 || len < 0) return einval(); + _offset2overlap(f->handle, off, &ov); if (l->l_type == F_RDLCK || l->l_type == F_WRLCK) { flags = 0; - if (cmd != F_SETLKW) { - flags |= kNtLockfileFailImmediately; - } - if (l->l_type == F_WRLCK) { - flags |= kNtLockfileExclusiveLock; - } - e = errno; - ok = LockFileEx(f->handle, flags, 0, len, len >> 32, &ov); - if (cmd == F_GETLK) { - if (ok) { - l->l_type = F_UNLCK; - if (!UnlockFileEx(f->handle, 0, len, len >> 32, &ov)) { - notpossible; - } - } else { - l->l_pid = -1; - ok = true; - } - } - if (ok) { - return 0; - } else if (__force_sqlite_to_work_until_we_can_fix_it) { - errno = e; + if (cmd == F_SETLK) flags |= kNtLockfileFailImmediately; + /* TODO: How can we make SQLite locks on Windows to work? */ + /* if (l->l_type == F_WRLCK) flags |= kNtLockfileExclusiveLock; */ + if (LockFileEx(f->handle, flags, 0, len, len >> 32, &ov)) { return 0; } else { + err = GetLastError(); + if (err == kNtErrorLockViolation) err = EAGAIN; + errno = err; return -1; } - } - - if (l->l_type == F_UNLCK) { - if (cmd == F_GETLK) return einval(); - e = errno; + } else if (l->l_type == F_UNLCK) { if (UnlockFileEx(f->handle, 0, len, len >> 32, &ov)) { return 0; - } else if (errno == ENOLCK) { - errno = e; - return 0; } else { - return 0; + err = GetLastError(); + if (err == kNtErrorNotLocked) { + return 0; + } else { + errno = err; + return -1; + } } + } else { + return einval(); } - - return einval(); } textwindows int sys_fcntl_nt(int fd, int cmd, uintptr_t arg) { diff --git a/libc/calls/fcntl-sysv.c b/libc/calls/fcntl-sysv.c index 2cfbf8d0c..66b398274 100644 --- a/libc/calls/fcntl-sysv.c +++ b/libc/calls/fcntl-sysv.c @@ -20,14 +20,12 @@ #include "libc/calls/syscall-sysv.internal.h" #include "libc/calls/syscall_support-sysv.internal.h" #include "libc/dce.h" -#include "libc/errno.h" #include "libc/intrin/asan.internal.h" #include "libc/sysv/consts/f.h" -#include "libc/sysv/consts/o.h" #include "libc/sysv/errfuns.h" int sys_fcntl(int fd, int cmd, uintptr_t arg) { - int e, rc; + int rc; bool islock; if ((islock = cmd == F_GETLK || // cmd == F_SETLK || // @@ -39,13 +37,9 @@ int sys_fcntl(int fd, int cmd, uintptr_t arg) { } cosmo2flock(arg); } - e = errno; rc = __sys_fcntl(fd, cmd, arg); if (islock) { flock2cosmo(arg); - } else if (rc == -1 && cmd == F_DUPFD_CLOEXEC && errno == EINVAL) { - errno = e; - rc = __fixupnewfd(__sys_fcntl(fd, F_DUPFD, arg), O_CLOEXEC); } return rc; } diff --git a/libc/calls/fstatat64.S b/libc/calls/fstatat64.S index 8b0c3e5f8..09086d2dd 100644 --- a/libc/calls/fstatat64.S +++ b/libc/calls/fstatat64.S @@ -1,5 +1,5 @@ -/*-*- mode:unix-assembly; indent-tabs-mode:t; tab-width:8; coding:utf-8 -*-│ -│vi: set et ft=asm ts=8 tw=8 fenc=utf-8 :vi│ +/*-*- mode:c;indent-tabs-mode:nil;c-basic-offset:2;tab-width:8;coding:utf-8 -*-│ +│vi: set net ft=c ts=2 sts=2 sw=2 fenc=utf-8 :vi│ ╞══════════════════════════════════════════════════════════════════════════════╡ │ This is free and unencumbered software released into the public domain. │ │ │ @@ -26,6 +26,5 @@ ╚─────────────────────────────────────────────────────────────────────────────*/ #include "libc/macros.internal.h" -fstatat64: - jmp fstatat +fstatat64: jmp fstatat .endfn fstatat64,globl diff --git a/libc/calls/ftruncate64.S b/libc/calls/ftruncate64.S index a4a2a5080..fbb6c5aa2 100644 --- a/libc/calls/ftruncate64.S +++ b/libc/calls/ftruncate64.S @@ -1,5 +1,5 @@ -/*-*- mode:unix-assembly; indent-tabs-mode:t; tab-width:8; coding:utf-8 -*-│ -│vi: set et ft=asm ts=8 tw=8 fenc=utf-8 :vi│ +/*-*- mode:c;indent-tabs-mode:nil;c-basic-offset:2;tab-width:8;coding:utf-8 -*-│ +│vi: set net ft=c ts=2 sts=2 sw=2 fenc=utf-8 :vi│ ╞══════════════════════════════════════════════════════════════════════════════╡ │ This is free and unencumbered software released into the public domain. │ │ │ @@ -26,6 +26,5 @@ ╚─────────────────────────────────────────────────────────────────────────────*/ #include "libc/macros.internal.h" -ftruncate64: - jmp ftruncate +ftruncate64: jmp ftruncate .endfn ftruncate64,globl diff --git a/libc/calls/g_sighandrvas.c b/libc/calls/g_sighandrvas.c index 3e0780d59..df628bc0e 100644 --- a/libc/calls/g_sighandrvas.c +++ b/libc/calls/g_sighandrvas.c @@ -17,7 +17,7 @@ │ PERFORMANCE OF THIS SOFTWARE. │ ╚─────────────────────────────────────────────────────────────────────────────*/ #include "libc/calls/state.internal.h" -#include "libc/thread/thread.h" +#include "libc/intrin/pthread.h" // TODO(jart): These should be _Thread_local but doing that currently // causes a regression with runitd.com on Windows. diff --git a/libc/calls/getloadavg-nt.c b/libc/calls/getloadavg-nt.c index 6a82d5bbc..1eba10b65 100644 --- a/libc/calls/getloadavg-nt.c +++ b/libc/calls/getloadavg-nt.c @@ -20,7 +20,7 @@ #include "libc/calls/syscall_support-nt.internal.h" #include "libc/dce.h" #include "libc/fmt/conv.h" -#include "libc/thread/thread.h" +#include "libc/intrin/pthread.h" #include "libc/macros.internal.h" #include "libc/nt/accounting.h" #include "libc/runtime/sysconf.h" diff --git a/libc/calls/interrupts-nt.c b/libc/calls/interrupts-nt.c index 749d0a192..82ccd8e6e 100644 --- a/libc/calls/interrupts-nt.c +++ b/libc/calls/interrupts-nt.c @@ -27,7 +27,7 @@ #include "libc/dce.h" #include "libc/intrin/lockcmpxchgp.h" #include "libc/intrin/weaken.h" -#include "libc/thread/tls.h" +#include "libc/nexgen32e/threaded.h" textwindows bool _check_interrupts(bool restartable, struct Fd *fd) { bool res; diff --git a/libc/calls/lstat64.S b/libc/calls/lstat64.S index 7cfe4b329..24d9d81e0 100644 --- a/libc/calls/lstat64.S +++ b/libc/calls/lstat64.S @@ -1,5 +1,5 @@ -/*-*- mode:unix-assembly; indent-tabs-mode:t; tab-width:8; coding:utf-8 -*-│ -│vi: set et ft=asm ts=8 tw=8 fenc=utf-8 :vi│ +/*-*- mode:c;indent-tabs-mode:nil;c-basic-offset:2;tab-width:8;coding:utf-8 -*-│ +│vi: set net ft=c ts=2 sts=2 sw=2 fenc=utf-8 :vi│ ╞══════════════════════════════════════════════════════════════════════════════╡ │ This is free and unencumbered software released into the public domain. │ │ │ @@ -26,6 +26,5 @@ ╚─────────────────────────────────────────────────────────────────────────────*/ #include "libc/macros.internal.h" -lstat64: - jmp lstat +lstat64: jmp lstat .endfn lstat64,globl diff --git a/libc/intrin/memtrack.greg.c b/libc/calls/memtrack.greg.c similarity index 99% rename from libc/intrin/memtrack.greg.c rename to libc/calls/memtrack.greg.c index 4f5a2482a..b35066329 100644 --- a/libc/intrin/memtrack.greg.c +++ b/libc/calls/memtrack.greg.c @@ -17,14 +17,14 @@ │ PERFORMANCE OF THIS SOFTWARE. │ ╚─────────────────────────────────────────────────────────────────────────────*/ #include "libc/assert.h" +#include "libc/intrin/bits.h" +#include "libc/intrin/likely.h" +#include "libc/intrin/weaken.h" #include "libc/calls/calls.h" #include "libc/calls/strace.internal.h" #include "libc/dce.h" #include "libc/errno.h" #include "libc/intrin/asan.internal.h" -#include "libc/intrin/bits.h" -#include "libc/intrin/likely.h" -#include "libc/intrin/weaken.h" #include "libc/log/libfatal.internal.h" #include "libc/log/log.h" #include "libc/macros.internal.h" @@ -201,9 +201,6 @@ int TrackMemoryInterval(struct MemoryIntervals *mm, int x, int y, long h, unsigned i; #if IsModeDbg() assert(y >= x); - if (!AreMemoryIntervalsOk(mm)) { - PrintMemoryIntervals(2, mm); - } assert(AreMemoryIntervalsOk(mm)); #endif diff --git a/libc/intrin/mman.greg.c b/libc/calls/mman.greg.c similarity index 98% rename from libc/intrin/mman.greg.c rename to libc/calls/mman.greg.c index 6e0cc8c47..fa0623e50 100644 --- a/libc/intrin/mman.greg.c +++ b/libc/calls/mman.greg.c @@ -74,7 +74,6 @@ noasan textreal uint64_t *__get_virtual(struct mman *mm, uint64_t *t, if (!(*e & PAGE_V)) { if (!maketables) return NULL; if (!(p = __new_page(mm))) return NULL; - __clear_page(BANE + p); *e = p | PAGE_V | PAGE_RW; } t = (uint64_t *)(BANE + (*e & PAGE_TA)); @@ -118,7 +117,7 @@ static noasan textreal void __invert_memory(struct mman *mm, uint64_t *pml4t) { uint64_t i, j, *m, p, pe; for (i = 0; i < mm->e820n; ++i) { for (p = mm->e820[i].addr, pe = mm->e820[i].addr + mm->e820[i].size; - p != pe + 0x200000; p += 4096) { + p + 0x200000 < pe; p += 4096) { m = __get_virtual(mm, pml4t, BANE + p, true); if (m && !(*m & PAGE_V)) { *m = p | PAGE_V | PAGE_RW; @@ -151,7 +150,7 @@ noasan textreal void __map_phdrs(struct mman *mm, uint64_t *pml4t, uint64_t b) { v = b + p->p_offset + i; m = MAX(m, v); } else { - v = __clear_page(BANE + __new_page(mm)); + v = __clear_page(__new_page(mm)); } *__get_virtual(mm, pml4t, p->p_vaddr + i, true) = (v & PAGE_TA) | f; } diff --git a/libc/calls/nanos.internal.h b/libc/calls/nanos.h similarity index 100% rename from libc/calls/nanos.internal.h rename to libc/calls/nanos.h diff --git a/libc/calls/preadv.c b/libc/calls/preadv.c index 349c00988..25ec37f4c 100644 --- a/libc/calls/preadv.c +++ b/libc/calls/preadv.c @@ -16,15 +16,12 @@ │ TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR │ │ PERFORMANCE OF THIS SOFTWARE. │ ╚─────────────────────────────────────────────────────────────────────────────*/ -#include "libc/assert.h" #include "libc/calls/calls.h" #include "libc/calls/internal.h" #include "libc/calls/state.internal.h" #include "libc/calls/strace.internal.h" #include "libc/calls/struct/iovec.h" #include "libc/calls/struct/iovec.internal.h" -#include "libc/calls/struct/sigset.h" -#include "libc/calls/struct/sigset.internal.h" #include "libc/calls/syscall-sysv.internal.h" #include "libc/dce.h" #include "libc/errno.h" @@ -36,89 +33,74 @@ #include "libc/intrin/weaken.h" #include "libc/macros.internal.h" #include "libc/sysv/consts/iov.h" -#include "libc/sysv/consts/sig.h" #include "libc/sysv/errfuns.h" #include "libc/zipos/zipos.internal.h" static ssize_t Preadv(int fd, struct iovec *iov, int iovlen, int64_t off) { - int e, i; - size_t got; - bool masked; - ssize_t rc, toto; - sigset_t mask, oldmask; - - if (fd < 0) { - return ebadf(); - } - - if (iovlen < 0) { - return einval(); - } - - if (IsAsan() && !__asan_is_valid_iov(iov, iovlen)) { - return efault(); - } + static bool once, demodernize; + int i, err; + ssize_t rc; + size_t got, toto; + if (fd < 0) return einval(); + if (iovlen < 0) return einval(); + if (IsAsan() && !__asan_is_valid_iov(iov, iovlen)) return efault(); if (fd < g_fds.n && g_fds.p[fd].kind == kFdZip) { return weaken(__zipos_read)( (struct ZiposHandle *)(intptr_t)g_fds.p[fd].handle, iov, iovlen, off); - } - - if (IsMetal()) { - return espipe(); // must be serial or console if not zipos - } - - if (IsWindows()) { + } else if (IsWindows()) { if (fd < g_fds.n) { return sys_read_nt(g_fds.p + fd, iov, iovlen, off); } else { return ebadf(); } - } - - while (iovlen && !iov->iov_len) { - --iovlen; - ++iov; - } - - if (!iovlen) { - return sys_pread(fd, 0, 0, off, off); + } else if (IsMetal()) { + return enosys(); } if (iovlen == 1) { - return sys_pread(fd, iov->iov_base, iov->iov_len, off, off); + return sys_pread(fd, iov[0].iov_base, iov[0].iov_len, off, off); } - e = errno; - rc = sys_preadv(fd, iov, iovlen, off, off); - if (rc != -1 || errno != ENOSYS) return rc; - errno = e; + /* + * NT, 2018-era XNU, and 2007-era Linux don't support this system call + */ + if (!__vforked && !once) { + err = errno; + rc = sys_preadv(fd, iov, iovlen, off, off); + if (rc == -1 && errno == ENOSYS) { + errno = err; + once = true; + demodernize = true; + STRACE("demodernizing %s() due to %s", "preadv", "ENOSYS"); + } else { + once = true; + return rc; + } + } + + if (!demodernize) { + return sys_preadv(fd, iov, iovlen, off, off); + } + + if (!iovlen) { + return sys_pread(fd, NULL, 0, off, off); + } for (toto = i = 0; i < iovlen; ++i) { rc = sys_pread(fd, iov[i].iov_base, iov[i].iov_len, off, off); if (rc == -1) { - if (!toto) { - toto = -1; + if (toto && (errno == EINTR || errno == EAGAIN)) { + return toto; + } else { + return -1; } - break; } - got = rc; toto += got; - off += got; if (got != iov[i].iov_len) { break; } - - if (!masked) { - sigfillset(&mask); - _npassert(!sys_sigprocmask(SIG_SETMASK, &mask, &oldmask)); - masked = true; - } - } - - if (masked) { - _npassert(!sys_sigprocmask(SIG_SETMASK, &oldmask, 0)); } return toto; diff --git a/libc/calls/pwritev.c b/libc/calls/pwritev.c index eda149625..120da1dba 100644 --- a/libc/calls/pwritev.c +++ b/libc/calls/pwritev.c @@ -16,13 +16,11 @@ │ TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR │ │ PERFORMANCE OF THIS SOFTWARE. │ ╚─────────────────────────────────────────────────────────────────────────────*/ -#include "libc/assert.h" #include "libc/calls/calls.h" #include "libc/calls/internal.h" #include "libc/calls/strace.internal.h" #include "libc/calls/struct/iovec.h" #include "libc/calls/struct/iovec.internal.h" -#include "libc/calls/struct/sigset.internal.h" #include "libc/calls/syscall-sysv.internal.h" #include "libc/dce.h" #include "libc/errno.h" @@ -33,90 +31,75 @@ #include "libc/intrin/weaken.h" #include "libc/macros.internal.h" #include "libc/sysv/consts/iov.h" -#include "libc/sysv/consts/sig.h" #include "libc/sysv/errfuns.h" #include "libc/zipos/zipos.internal.h" static ssize_t Pwritev(int fd, const struct iovec *iov, int iovlen, int64_t off) { - int i, e; - bool masked; - size_t sent; - ssize_t rc, toto; - sigset_t mask, oldmask; - - if (fd < 0) { - return ebadf(); - } - - if (iovlen < 0) { - return einval(); - } - - if (IsAsan() && !__asan_is_valid_iov(iov, iovlen)) { - return efault(); - } + static bool once, demodernize; + int i, err; + ssize_t rc; + size_t sent, toto; + if (fd < 0) return einval(); + if (iovlen < 0) return einval(); + if (IsAsan() && !__asan_is_valid_iov(iov, iovlen)) return efault(); if (fd < g_fds.n && g_fds.p[fd].kind == kFdZip) { return weaken(__zipos_write)( (struct ZiposHandle *)(intptr_t)g_fds.p[fd].handle, iov, iovlen, off); - } - - if (IsWindows()) { + } else if (IsWindows()) { if (fd < g_fds.n) { return sys_write_nt(fd, iov, iovlen, off); } else { return ebadf(); } - } - - if (IsMetal()) { - return espipe(); // must be serial or console if not zipos - } - - while (iovlen && !iov->iov_len) { - --iovlen; - ++iov; - } - - if (!iovlen) { - return sys_pwrite(fd, 0, 0, off, off); + } else if (IsMetal()) { + return enosys(); } if (iovlen == 1) { - return sys_pwrite(fd, iov->iov_base, iov->iov_len, off, off); + return sys_pwrite(fd, iov[0].iov_base, iov[0].iov_len, off, off); } - e = errno; - rc = sys_pwritev(fd, iov, iovlen, off, off); - if (rc != -1 || errno != ENOSYS) return rc; - errno = e; + /* + * NT, 2018-era XNU, and 2007-era Linux don't support this system call + */ + if (!once) { + err = errno; + rc = sys_pwritev(fd, iov, iovlen, off, off); + if (rc == -1 && errno == ENOSYS) { + errno = err; + once = true; + demodernize = true; + STRACE("demodernizing %s() due to %s", "pwritev", "ENOSYS"); + } else { + once = true; + return rc; + } + } + + if (!demodernize) { + return sys_pwritev(fd, iov, iovlen, off, off); + } + + if (!iovlen) { + return sys_pwrite(fd, NULL, 0, off, off); + } for (toto = i = 0; i < iovlen; ++i) { rc = sys_pwrite(fd, iov[i].iov_base, iov[i].iov_len, off, off); if (rc == -1) { - if (!toto) { - toto = -1; + if (toto && (errno == EINTR || errno == EAGAIN)) { + return toto; + } else { + return -1; } - break; } - sent = rc; toto += sent; - off += sent; if (sent != iov[i].iov_len) { break; } - - if (!masked) { - sigfillset(&mask); - _npassert(!sys_sigprocmask(SIG_SETMASK, &mask, &oldmask)); - masked = true; - } - } - - if (masked) { - _npassert(!sys_sigprocmask(SIG_SETMASK, &oldmask, 0)); } return toto; diff --git a/libc/calls/raise.c b/libc/calls/raise.c index b94c49e55..d2fb4927b 100644 --- a/libc/calls/raise.c +++ b/libc/calls/raise.c @@ -21,7 +21,7 @@ #include "libc/calls/strace.internal.h" #include "libc/calls/syscall-sysv.internal.h" #include "libc/dce.h" -#include "libc/thread/tls.h" +#include "libc/nexgen32e/threaded.h" #include "libc/runtime/internal.h" #include "libc/sysv/consts/sicode.h" #include "libc/sysv/consts/sig.h" diff --git a/libc/calls/readv-metal.c b/libc/calls/readv-metal.c index 3e283a1f3..bb2cf19bb 100644 --- a/libc/calls/readv-metal.c +++ b/libc/calls/readv-metal.c @@ -20,28 +20,15 @@ #include "libc/calls/struct/fd.internal.h" #include "libc/calls/struct/iovec.h" #include "libc/calls/struct/iovec.internal.h" -#include "libc/intrin/weaken.h" #include "libc/macros.internal.h" #include "libc/str/str.h" #include "libc/sysv/errfuns.h" -#include "libc/vga/vga.internal.h" ssize_t sys_readv_metal(struct Fd *fd, const struct iovec *iov, int iovlen) { int i; size_t got, toto; struct MetalFile *file; switch (fd->kind) { - case kFdConsole: - /* - * The VGA teletypewriter code may wish to send out "status report" - * escape sequences, in response to requests sent to it via write(). - * Read & return these if they are available. - */ - if (weaken(sys_readv_vga)) { - ssize_t res = weaken(sys_readv_vga)(fd, iov, iovlen); - if (res > 0) return res; - } - /* fall through */ case kFdSerial: return sys_readv_serial(fd, iov, iovlen); case kFdFile: diff --git a/libc/calls/reservefd.c b/libc/calls/reservefd.c index d0a4f3e34..ee27f7b6f 100644 --- a/libc/calls/reservefd.c +++ b/libc/calls/reservefd.c @@ -18,7 +18,6 @@ ╚─────────────────────────────────────────────────────────────────────────────*/ #include "libc/assert.h" #include "libc/calls/calls.h" -#include "libc/calls/extend.internal.h" #include "libc/calls/internal.h" #include "libc/calls/state.internal.h" #include "libc/calls/strace.internal.h" @@ -47,11 +46,33 @@ static volatile size_t mapsize; * @asyncsignalsafe */ int __ensurefds_unlocked(int fd) { - bool relocate; + uint64_t addr; + int prot, flags; + size_t size, chunk; + struct DirectMap dm; if (fd < g_fds.n) return fd; - g_fds.n = fd + 1; - g_fds.e = - _extend(g_fds.p, g_fds.n * sizeof(*g_fds.p), g_fds.e, 0x6ff000000000); + STRACE("__ensurefds(%d) extending", fd); + size = mapsize; + chunk = FRAMESIZE; + if (IsAsan()) chunk *= 8; + addr = kMemtrackFdsStart + size; + prot = PROT_READ | PROT_WRITE; + flags = MAP_PRIVATE | MAP_ANONYMOUS | MAP_FIXED; + dm = sys_mmap((char *)addr, chunk, prot, flags, -1, 0); + TrackMemoryInterval(&_mmi, addr >> 16, (addr + chunk - 1) >> 16, dm.maphandle, + prot, flags, false, false, 0, chunk); + if (IsAsan()) { + addr = (addr >> 3) + 0x7fff8000; + dm = sys_mmap((char *)addr, FRAMESIZE, prot, flags, -1, 0); + TrackMemoryInterval(&_mmi, addr >> 16, addr >> 16, dm.maphandle, prot, + flags, false, false, 0, FRAMESIZE); + } + if (!size) { + g_fds.p = memcpy((char *)kMemtrackFdsStart, g_fds.__init_p, + sizeof(g_fds.__init_p)); + } + g_fds.n = (size + chunk) / sizeof(*g_fds.p); + mapsize = size + chunk; return fd; } diff --git a/libc/calls/samplepids.c b/libc/calls/samplepids.c index b1a6c8016..035f43356 100644 --- a/libc/calls/samplepids.c +++ b/libc/calls/samplepids.c @@ -19,8 +19,8 @@ #include "libc/assert.h" #include "libc/calls/internal.h" #include "libc/dce.h" -#include "libc/thread/thread.h" -#include "libc/thread/tls.h" +#include "libc/intrin/pthread.h" +#include "libc/nexgen32e/threaded.h" #include "libc/stdio/lcg.internal.h" /** diff --git a/libc/calls/sched_setscheduler.c b/libc/calls/sched_setscheduler.c index e5fb93996..211f14326 100644 --- a/libc/calls/sched_setscheduler.c +++ b/libc/calls/sched_setscheduler.c @@ -22,7 +22,7 @@ #include "libc/calls/struct/sched_param.internal.h" #include "libc/dce.h" #include "libc/intrin/describeflags.internal.h" -#include "libc/thread/tls.h" +#include "libc/nexgen32e/threaded.h" #include "libc/str/str.h" #include "libc/sysv/errfuns.h" diff --git a/libc/calls/semaphore.internal.h b/libc/calls/semaphore.internal.h deleted file mode 100644 index 71e7d6602..000000000 --- a/libc/calls/semaphore.internal.h +++ /dev/null @@ -1,26 +0,0 @@ -#ifndef COSMOPOLITAN_LIBC_CALLS_SEMAPHORE_H_ -#define COSMOPOLITAN_LIBC_CALLS_SEMAPHORE_H_ -#include "libc/calls/struct/timespec.h" -#if !(__ASSEMBLER__ + __LINKER__ + 0) -COSMOPOLITAN_C_START_ - -#define SEM_FAILED ((sem_t *)0) - -typedef struct { - volatile int __val[4 * sizeof(long) / sizeof(int)]; -} sem_t; - -int sem_close(sem_t *); -int sem_destroy(sem_t *); -int sem_getvalue(sem_t *, int *); -int sem_init(sem_t *, int, unsigned); -sem_t *sem_open(const char *, int, ...); -int sem_post(sem_t *); -int sem_timedwait(sem_t *, const struct timespec *); -int sem_trywait(sem_t *); -int sem_unlink(const char *); -int sem_wait(sem_t *); - -COSMOPOLITAN_C_END_ -#endif /* !(__ASSEMBLER__ + __LINKER__ + 0) */ -#endif /* COSMOPOLITAN_LIBC_CALLS_SEMAPHORE_H_ */ diff --git a/libc/calls/sig2.c b/libc/calls/sig2.c index 9d4c0bd19..606514086 100644 --- a/libc/calls/sig2.c +++ b/libc/calls/sig2.c @@ -16,7 +16,6 @@ │ TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR │ │ PERFORMANCE OF THIS SOFTWARE. │ ╚─────────────────────────────────────────────────────────────────────────────*/ -#include "libc/calls/calls.h" #include "libc/calls/sig.internal.h" #include "libc/calls/state.internal.h" #include "libc/calls/strace.internal.h" diff --git a/libc/calls/siglock.c b/libc/calls/siglock.c index 9d3632dea..c9feb3ba2 100644 --- a/libc/calls/siglock.c +++ b/libc/calls/siglock.c @@ -17,7 +17,7 @@ │ PERFORMANCE OF THIS SOFTWARE. │ ╚─────────────────────────────────────────────────────────────────────────────*/ #include "libc/calls/state.internal.h" -#include "libc/thread/thread.h" +#include "libc/intrin/pthread.h" static pthread_mutex_t __sig_lock_obj; @@ -30,5 +30,5 @@ void(__sig_unlock)(void) { } __attribute__((__constructor__)) static void init(void) { - __sig_lock_obj._type = PTHREAD_MUTEX_RECURSIVE; + __sig_lock_obj.type = PTHREAD_MUTEX_RECURSIVE; } diff --git a/libc/calls/stat64.S b/libc/calls/stat64.S index e61b7bb04..7b6738b40 100644 --- a/libc/calls/stat64.S +++ b/libc/calls/stat64.S @@ -1,5 +1,5 @@ -/*-*- mode:unix-assembly; indent-tabs-mode:t; tab-width:8; coding:utf-8 -*-│ -│vi: set et ft=asm ts=8 tw=8 fenc=utf-8 :vi│ +/*-*- mode:c;indent-tabs-mode:nil;c-basic-offset:2;tab-width:8;coding:utf-8 -*-│ +│vi: set net ft=c ts=2 sts=2 sw=2 fenc=utf-8 :vi│ ╞══════════════════════════════════════════════════════════════════════════════╡ │ This is free and unencumbered software released into the public domain. │ │ │ diff --git a/libc/calls/state.internal.h b/libc/calls/state.internal.h index 4f9a4dc3b..b5eacf1ba 100644 --- a/libc/calls/state.internal.h +++ b/libc/calls/state.internal.h @@ -1,8 +1,8 @@ #ifndef COSMOPOLITAN_LIBC_CALLS_STATE_INTERNAL_H_ #define COSMOPOLITAN_LIBC_CALLS_STATE_INTERNAL_H_ -#include "libc/intrin/nopl.internal.h" -#include "libc/thread/thread.h" -#include "libc/thread/tls.h" +#include "libc/intrin/nopl.h" +#include "libc/intrin/pthread.h" +#include "libc/nexgen32e/threaded.h" #if !(__ASSEMBLER__ + __LINKER__ + 0) COSMOPOLITAN_C_START_ diff --git a/libc/calls/struct/fd.internal.h b/libc/calls/struct/fd.internal.h index fe117b253..17d586236 100644 --- a/libc/calls/struct/fd.internal.h +++ b/libc/calls/struct/fd.internal.h @@ -25,9 +25,10 @@ struct Fd { }; struct Fds { - int f; /* lowest free slot */ - size_t n; - struct Fd *p, *e; + int f; /* lowest free slot */ + size_t n; /* monotonic capacity */ + struct Fd *p; + struct Fd __init_p[OPEN_MAX]; }; COSMOPOLITAN_C_END_ diff --git a/libc/calls/struct/timespec.h b/libc/calls/struct/timespec.h index 3bc0cb042..4c555e637 100644 --- a/libc/calls/struct/timespec.h +++ b/libc/calls/struct/timespec.h @@ -17,7 +17,6 @@ int utimensat(int, const char *, const struct timespec[2], int); int timespec_get(struct timespec *, int); int timespec_getres(struct timespec *, int); -int _timespec_cmp(struct timespec, struct timespec) pureconst; bool _timespec_eq(struct timespec, struct timespec) pureconst; bool _timespec_gte(struct timespec, struct timespec) pureconst; int64_t _timespec_tomicros(struct timespec) pureconst; diff --git a/libc/calls/symlinkat-nt.c b/libc/calls/symlinkat-nt.c index cdb7acc49..dd36786bd 100644 --- a/libc/calls/symlinkat-nt.c +++ b/libc/calls/symlinkat-nt.c @@ -19,7 +19,7 @@ #include "libc/calls/calls.h" #include "libc/calls/syscall_support-nt.internal.h" #include "libc/errno.h" -#include "libc/thread/thread.h" +#include "libc/intrin/pthread.h" #include "libc/nt/enum/accessmask.h" #include "libc/nt/enum/fileflagandattributes.h" #include "libc/nt/enum/symboliclink.h" diff --git a/libc/calls/unveil.c b/libc/calls/unveil.c index b439553b5..759ad3db8 100644 --- a/libc/calls/unveil.c +++ b/libc/calls/unveil.c @@ -30,7 +30,7 @@ #include "libc/errno.h" #include "libc/fmt/conv.h" #include "libc/macros.internal.h" -#include "libc/thread/tls.h" +#include "libc/nexgen32e/threaded.h" #include "libc/runtime/internal.h" #include "libc/runtime/runtime.h" #include "libc/runtime/stack.h" diff --git a/libc/dns/gethoststxt.c b/libc/dns/gethoststxt.c index 55fb566a7..32ff8dfce 100644 --- a/libc/dns/gethoststxt.c +++ b/libc/dns/gethoststxt.c @@ -23,7 +23,7 @@ #include "libc/dce.h" #include "libc/dns/hoststxt.h" #include "libc/fmt/fmt.h" -#include "libc/thread/thread.h" +#include "libc/intrin/pthread.h" #include "libc/macros.internal.h" #include "libc/nt/systeminfo.h" #include "libc/runtime/runtime.h" diff --git a/libc/dns/getresolvconf.c b/libc/dns/getresolvconf.c index 1546ee9c6..e226ac13f 100644 --- a/libc/dns/getresolvconf.c +++ b/libc/dns/getresolvconf.c @@ -20,7 +20,7 @@ #include "libc/dce.h" #include "libc/dns/resolvconf.h" #include "libc/fmt/fmt.h" -#include "libc/thread/thread.h" +#include "libc/intrin/pthread.h" #include "libc/macros.internal.h" #include "libc/runtime/runtime.h" #include "libc/sock/sock.h" diff --git a/libc/fmt/conv.h b/libc/fmt/conv.h index 71fb95067..630cbac7b 100644 --- a/libc/fmt/conv.h +++ b/libc/fmt/conv.h @@ -118,8 +118,7 @@ imaxdiv_t imaxdiv(intmax_t, intmax_t) pureconst; #define lldiv(num, den) ((lldiv_t){(num) / (den), (num) % (den)}) #endif -#if (__GNUC__ * 100 + __GNUC_MINOR__ >= 406 || defined(__llvm__)) && \ - !defined(__STRICT_ANSI__) +#if __GNUC__ * 100 + __GNUC_MINOR__ >= 406 || defined(__llvm__) int128_t i128abs(int128_t) libcesque pureconst; int128_t strtoi128(const char *, char **, int) paramsnonnull((1)); uint128_t strtou128(const char *, char **, int) paramsnonnull((1)); diff --git a/libc/intrin/lengthuint64.c b/libc/fmt/lengthuint64.c similarity index 100% rename from libc/intrin/lengthuint64.c rename to libc/fmt/lengthuint64.c diff --git a/libc/integral/c.inc b/libc/integral/c.inc index cfc14baf5..68bcd7090 100644 --- a/libc/integral/c.inc +++ b/libc/integral/c.inc @@ -123,9 +123,7 @@ typedef __UINT64_TYPE__ uint64_t; typedef __INTMAX_TYPE__ intmax_t; typedef __UINTMAX_TYPE__ uintmax_t; -#if ((__GNUC__ + 0) * 100 + (__GNUC_MINOR__ + 0) >= 406 || \ - defined(__llvm__)) && \ - !defined(__STRICT_ANSI__) +#if (__GNUC__ + 0) * 100 + (__GNUC_MINOR__ + 0) >= 406 || defined(__llvm__) typedef signed __int128 int128_t; typedef unsigned __int128 uint128_t; #endif diff --git a/libc/integral/normalize.inc b/libc/integral/normalize.inc index 30be7c9b9..513778444 100644 --- a/libc/integral/normalize.inc +++ b/libc/integral/normalize.inc @@ -67,8 +67,8 @@ #endif /* TODO(jart): Remove this in favor of GetStackSize() */ -#if defined(COSMO) && (defined(MODE_DBG) || defined(__SANITIZE_ADDRESS__)) -#define STACKSIZE 262144 /* 256kb stack */ +#if defined(COSMO) && defined(MODE_DBG) +#define STACKSIZE 131072 /* 128kb stack */ #elif defined(COSMO) #define STACKSIZE 65536 /* 64kb stack */ #else diff --git a/libc/intrin/asan.c b/libc/intrin/asan.c index dc0315568..5fb061e14 100644 --- a/libc/intrin/asan.c +++ b/libc/intrin/asan.c @@ -25,10 +25,10 @@ #include "libc/intrin/asan.internal.h" #include "libc/intrin/asancodes.h" #include "libc/intrin/kprintf.h" -#include "libc/intrin/leaky.internal.h" #include "libc/intrin/likely.h" #include "libc/intrin/lockcmpxchg.h" #include "libc/intrin/nomultics.internal.h" +#include "libc/intrin/pthread.h" #include "libc/intrin/weaken.h" #include "libc/log/backtrace.internal.h" #include "libc/log/internal.h" @@ -39,7 +39,9 @@ #include "libc/mem/mem.h" #include "libc/mem/reverse.internal.h" #include "libc/nexgen32e/gc.internal.h" +#include "libc/nexgen32e/gettls.h" #include "libc/nexgen32e/stackframe.h" +#include "libc/nexgen32e/threaded.h" #include "libc/nt/enum/version.h" #include "libc/nt/runtime.h" #include "libc/runtime/directmap.internal.h" @@ -56,7 +58,6 @@ #include "libc/sysv/consts/prot.h" #include "libc/sysv/errfuns.h" #include "libc/thread/thread.h" -#include "libc/thread/tls.h" #include "third_party/dlmalloc/dlmalloc.h" STATIC_YOINK("_init_asan"); @@ -171,7 +172,7 @@ struct ReportOriginHeap { }; static int __asan_noreentry; -static pthread_spinlock_t __asan_lock; +static pthread_mutex_t __asan_lock; static struct AsanMorgue __asan_morgue; #define __asan_unreachable() \ @@ -868,25 +869,25 @@ dontdiscard __asan_die_f *__asan_report_memory_fault(void *addr, int size, void *__asan_morgue_add(void *p) { int i; void *r; - if (__threaded) pthread_spin_lock(&__asan_lock); + if (__threaded) pthread_mutex_lock(&__asan_lock); i = __asan_morgue.i++ & (ARRAYLEN(__asan_morgue.p) - 1); r = __asan_morgue.p[i]; __asan_morgue.p[i] = p; - if (__threaded) pthread_spin_unlock(&__asan_lock); + if (__threaded) pthread_mutex_unlock(&__asan_lock); return r; } static void __asan_morgue_flush(void) { int i; void *p; - if (__threaded) pthread_spin_lock(&__asan_lock); + if (__threaded) pthread_mutex_lock(&__asan_lock); for (i = 0; i < ARRAYLEN(__asan_morgue.p); ++i) { if (__asan_morgue.p[i] && weaken(dlfree)) { weaken(dlfree)(__asan_morgue.p[i]); } __asan_morgue.p[i] = 0; } - if (__threaded) pthread_spin_unlock(&__asan_lock); + if (__threaded) pthread_mutex_unlock(&__asan_lock); } static size_t __asan_user_size(size_t n) { @@ -937,7 +938,7 @@ static void __asan_trace(struct AsanTrace *bt, const struct StackFrame *bp) { size_t i, gi; intptr_t addr; struct Garbages *garbage; - garbage = __tls_enabled ? __get_tls()->tib_garbages : 0; + garbage = __tls_enabled ? ((cthread_t)__get_tls())->garbages : 0; gi = garbage ? garbage->i : 0; for (f1 = -1, i = 0; bp && i < ARRAYLEN(bt->p); ++i, bp = bp->next) { if (f1 != (f2 = ((intptr_t)bp >> 16))) { @@ -1053,30 +1054,6 @@ int __asan_print_trace(void *p) { return 0; } -// Returns true if `p` was allocated by an IGNORE_LEAKS(function). -int __asan_is_leaky(void *p) { - int sym; - size_t c, i, n; - intptr_t f, *l; - struct AsanExtra *e; - struct SymbolTable *st; - if (!weaken(GetSymbolTable)) notpossible; - if (!(e = __asan_get_extra(p, &c))) return 0; - if (!__asan_read48(e->size, &n)) return 0; - if (!__asan_is_mapped((((intptr_t)p >> 3) + 0x7fff8000) >> 16)) return 0; - if (!(st = GetSymbolTable())) return 0; - for (i = 0; i < ARRAYLEN(e->bt.p) && e->bt.p[i]; ++i) { - if ((sym = weaken(__get_symbol)(st, e->bt.p[i])) == -1) continue; - f = st->addr_base + st->symbols[sym].x; - for (l = _leaky_start; l < _leaky_end; ++l) { - if (f == *l) { - return 1; - } - } - } - return 0; -} - static void __asan_deallocate(char *p, long kind) { size_t c, n; struct AsanExtra *e; diff --git a/libc/intrin/asan.internal.h b/libc/intrin/asan.internal.h index eee7b4797..5dc92372f 100644 --- a/libc/intrin/asan.internal.h +++ b/libc/intrin/asan.internal.h @@ -3,7 +3,6 @@ #include "libc/calls/struct/iovec.h" #include "libc/intrin/asancodes.h" #include "libc/macros.internal.h" -#include "libc/thread/thread.h" #if !(__ASSEMBLER__ + __LINKER__ + 0) COSMOPOLITAN_C_START_ @@ -30,7 +29,6 @@ struct AsanFault __asan_check(const void *, long) nosideeffect; void __asan_free(void *); void *__asan_malloc(size_t); -int __asan_is_leaky(void *); int __asan_malloc_trim(size_t); int __asan_print_trace(void *); void *__asan_calloc(size_t, size_t); diff --git a/libc/intrin/bigword.internal.h b/libc/intrin/bigword.internal.h new file mode 100644 index 000000000..4bf33dced --- /dev/null +++ b/libc/intrin/bigword.internal.h @@ -0,0 +1,16 @@ +#ifndef COSMOPOLITAN_LIBC_BITS_BIGWORD_H_ +#define COSMOPOLITAN_LIBC_BITS_BIGWORD_H_ + +#ifndef BIGWORD +#if __AVX512F__ + 0 +#define BIGWORD 64 +#elif __AVX2__ + 0 +#define BIGWORD 32 +#elif __SSE2__ + 0 +#define BIGWORD 16 +#else +#define BIGWORD __BIGGEST_ALIGNMENT__ +#endif +#endif /*BIGWORD*/ + +#endif /* COSMOPOLITAN_LIBC_BITS_BIGWORD_H_ */ diff --git a/libc/thread/pthread_cond_signal.c b/libc/intrin/cmpxchg.c similarity index 64% rename from libc/thread/pthread_cond_signal.c rename to libc/intrin/cmpxchg.c index 11dfd11c4..f8e363138 100644 --- a/libc/thread/pthread_cond_signal.c +++ b/libc/intrin/cmpxchg.c @@ -1,7 +1,7 @@ /*-*- mode:c;indent-tabs-mode:nil;c-basic-offset:2;tab-width:8;coding:utf-8 -*-│ │vi: set net ft=c ts=2 sts=2 sw=2 fenc=utf-8 :vi│ ╞══════════════════════════════════════════════════════════════════════════════╡ -│ Copyright 2022 Justine Alexandra Roberts Tunney │ +│ Copyright 2021 Justine Alexandra Roberts Tunney │ │ │ │ Permission to use, copy, modify, and/or distribute this software for │ │ any purpose with or without fee is hereby granted, provided that the │ @@ -16,27 +16,32 @@ │ TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR │ │ PERFORMANCE OF THIS SOFTWARE. │ ╚─────────────────────────────────────────────────────────────────────────────*/ -#include "libc/thread/thread.h" -#include "third_party/nsync/cv.h" +#include "libc/intrin/cmpxchg.h" /** - * Wakes at least one thread waiting on condition, e.g. + * Compares and exchanges. * - * pthread_cond_t cond = PTHREAD_COND_INITIALIZER; - * pthread_mutex_t lock = PTHREAD_MUTEX_INITIALIZER; - * // ... - * pthread_mutex_lock(&lock); - * pthread_cond_signal(&cond, &lock); - * pthread_mutex_unlock(&lock); - * - * This function has no effect if there aren't any threads currently - * waiting on the condition. - * - * @return 0 on success, or errno on error - * @see pthread_cond_broadcast - * @see pthread_cond_wait + * @param ifthing is uint𝑘_t[hasatleast 1] where 𝑘 ∈ {8,16,32,64} + * @param size is automatically supplied by macro wrapper + * @return true if value was exchanged, otherwise false + * @see _lockcmpxchg() */ -int pthread_cond_signal(pthread_cond_t *cond) { - nsync_cv_signal((nsync_cv *)cond); - return 0; +bool(_cmpxchg)(void *ifthing, intptr_t isequaltome, intptr_t replaceitwithme, + size_t size) { + switch (size) { + case 1: + return _cmpxchg((int8_t *)ifthing, (int8_t)isequaltome, + (int8_t)replaceitwithme); + case 2: + return _cmpxchg((int16_t *)ifthing, (int16_t)isequaltome, + (int16_t)replaceitwithme); + case 4: + return _cmpxchg((int32_t *)ifthing, (int32_t)isequaltome, + (int32_t)replaceitwithme); + case 8: + return _cmpxchg((int64_t *)ifthing, (int64_t)isequaltome, + (int64_t)replaceitwithme); + default: + return false; + } } diff --git a/libc/intrin/cmpxchg.h b/libc/intrin/cmpxchg.h index 54e45d5dc..3ba5a734a 100644 --- a/libc/intrin/cmpxchg.h +++ b/libc/intrin/cmpxchg.h @@ -4,6 +4,8 @@ #if !(__ASSEMBLER__ + __LINKER__ + 0) COSMOPOLITAN_C_START_ +bool _cmpxchg(void *, intptr_t, intptr_t, size_t); + #if defined(__GNUC__) && !defined(__STRICT_ANSI__) && defined(__x86__) #define _cmpxchg(IFTHING, ISEQUALTOME, REPLACEITWITHME) \ ({ \ @@ -17,6 +19,9 @@ COSMOPOLITAN_C_START_ : "cc"); \ DidIt; \ }) +#else +#define _cmpxchg(MEM, CMP, VAL) \ + _cmpxchg(MEM, (intptr_t)(CMP), (intptr_t)(VAL), sizeof(*(MEM))) #endif /* GNUC && !ANSI && x86 */ COSMOPOLITAN_C_END_ diff --git a/libc/intrin/cxaatexit.internal.h b/libc/intrin/cxaatexit.internal.h index bf96a4e47..99e1e9036 100644 --- a/libc/intrin/cxaatexit.internal.h +++ b/libc/intrin/cxaatexit.internal.h @@ -1,7 +1,7 @@ #ifndef COSMOPOLITAN_LIBC_RUNTIME_CXAATEXIT_H_ #define COSMOPOLITAN_LIBC_RUNTIME_CXAATEXIT_H_ -#include "libc/intrin/nopl.internal.h" -#include "libc/thread/tls.h" +#include "libc/intrin/nopl.h" +#include "libc/nexgen32e/threaded.h" #include "libc/stdio/stdio.h" #if !(__ASSEMBLER__ + __LINKER__ + 0) COSMOPOLITAN_C_START_ diff --git a/libc/intrin/cxalock.c b/libc/intrin/cxalock.c index 1f7b8d652..49b2d96b6 100644 --- a/libc/intrin/cxalock.c +++ b/libc/intrin/cxalock.c @@ -17,7 +17,7 @@ │ PERFORMANCE OF THIS SOFTWARE. │ ╚─────────────────────────────────────────────────────────────────────────────*/ #include "libc/intrin/cxaatexit.internal.h" -#include "libc/thread/thread.h" +#include "libc/intrin/pthread.h" static pthread_mutex_t __cxa_lock_obj; diff --git a/libc/intrin/describeflags.internal.h b/libc/intrin/describeflags.internal.h index 8ec3b0aa5..0bc065de1 100644 --- a/libc/intrin/describeflags.internal.h +++ b/libc/intrin/describeflags.internal.h @@ -17,7 +17,7 @@ const char *DescribeCapability(char[20], int); const char *DescribeClockName(char[32], int); const char *DescribeDirfd(char[12], int); const char *DescribeFrame(char[32], int); -const char *DescribeFutexOp(char[64], int); +const char *DescribeFutexOp(int); const char *DescribeFutexResult(char[12], int); const char *DescribeHow(char[12], int); const char *DescribeMapFlags(char[64], int); @@ -30,7 +30,6 @@ const char *DescribeNtFileFlagAttr(char[256], uint32_t); const char *DescribeNtFileMapFlags(char[64], uint32_t); const char *DescribeNtFileShareFlags(char[64], uint32_t); const char *DescribeNtFiletypeFlags(char[64], uint32_t); -const char *DescribeNtLockFileFlags(char[64], uint32_t); const char *DescribeNtMovFileInpFlags(char[256], uint32_t); const char *DescribeNtPageFlags(char[64], uint32_t); const char *DescribeNtPipeModeFlags(char[64], uint32_t); @@ -61,7 +60,6 @@ const char *DescribeWhence(char[12], int); #define DescribeClockName(x) DescribeClockName(alloca(32), x) #define DescribeDirfd(x) DescribeDirfd(alloca(12), x) #define DescribeFrame(x) DescribeFrame(alloca(32), x) -#define DescribeFutexOp(x) DescribeFutexOp(alloca(64), x) #define DescribeFutexResult(x) DescribeFutexResult(alloca(12), x) #define DescribeHow(x) DescribeHow(alloca(12), x) #define DescribeMapFlags(x) DescribeMapFlags(alloca(64), x) @@ -73,7 +71,6 @@ const char *DescribeWhence(char[12], int); #define DescribeNtFileMapFlags(x) DescribeNtFileMapFlags(alloca(64), x) #define DescribeNtFileShareFlags(x) DescribeNtFileShareFlags(alloca(64), x) #define DescribeNtFiletypeFlags(x) DescribeNtFiletypeFlags(alloca(64), x) -#define DescribeNtLockFileFlags(x) DescribeNtLockFileFlags(alloca(64), x) #define DescribeNtMovFileInpFlags(x) DescribeNtMovFileInpFlags(alloca(256), x) #define DescribeNtPageFlags(x) DescribeNtPageFlags(alloca(64), x) #define DescribeNtPipeModeFlags(x) DescribeNtPipeModeFlags(alloca(64), x) diff --git a/libc/intrin/describefutexop.c b/libc/intrin/describefutexop.c index 80993d8d7..f956658d0 100644 --- a/libc/intrin/describefutexop.c +++ b/libc/intrin/describefutexop.c @@ -16,47 +16,16 @@ │ TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR │ │ PERFORMANCE OF THIS SOFTWARE. │ ╚─────────────────────────────────────────────────────────────────────────────*/ -#include "libc/fmt/itoa.h" #include "libc/intrin/describeflags.internal.h" -#include "libc/str/str.h" #include "libc/sysv/consts/futex.h" -const char *(DescribeFutexOp)(char buf[64], int x) { - - bool priv = false; - if (x & FUTEX_PRIVATE_FLAG) { - priv = true; - x &= ~FUTEX_PRIVATE_FLAG; - } - - bool real = false; - if (x & FUTEX_CLOCK_REALTIME) { - real = true; - x &= ~FUTEX_CLOCK_REALTIME; - } - - char *p = buf; - - if (x == FUTEX_WAIT) { - p = stpcpy(p, "FUTEX_WAIT"); - } else if (x == FUTEX_WAKE) { - p = stpcpy(p, "FUTEX_WAKE"); - } else if (x == FUTEX_REQUEUE) { - p = stpcpy(p, "FUTEX_REQUEUE"); - } else if (x == FUTEX_WAIT_BITSET) { - p = stpcpy(p, "FUTEX_WAIT_BITSET"); - } else { - p = stpcpy(p, "FUTEX_"); - p = FormatUint32(p, x); - } - - if (priv) { - p = stpcpy(p, "_PRIVATE"); - } - - if (real) { - p = stpcpy(p, "|FUTEX_CLOCK_REALTIME"); - } - - return buf; +const char *DescribeFutexOp(int x) { + if (x == FUTEX_WAIT) return "FUTEX_WAIT"; + if (x == FUTEX_WAKE) return "FUTEX_WAKE"; + if (x == FUTEX_REQUEUE) return "FUTEX_REQUEUE"; + // order matters (the private bit might be zero) + if (x == FUTEX_WAIT_PRIVATE) return "FUTEX_WAIT_PRIVATE"; + if (x == FUTEX_WAKE_PRIVATE) return "FUTEX_WAKE_PRIVATE"; + if (x == FUTEX_REQUEUE_PRIVATE) return "FUTEX_REQUEUE_PRIVATE"; + return "FUTEX_???"; } diff --git a/libc/intrin/describentoverlapped.c b/libc/intrin/describentoverlapped.c deleted file mode 100644 index 3a4b33a74..000000000 --- a/libc/intrin/describentoverlapped.c +++ /dev/null @@ -1,50 +0,0 @@ -/*-*- mode:c;indent-tabs-mode:nil;c-basic-offset:2;tab-width:8;coding:utf-8 -*-│ -│vi: set net ft=c ts=2 sts=2 sw=2 fenc=utf-8 :vi│ -╞══════════════════════════════════════════════════════════════════════════════╡ -│ Copyright 2022 Justine Alexandra Roberts Tunney │ -│ │ -│ Permission to use, copy, modify, and/or distribute this software for │ -│ any purpose with or without fee is hereby granted, provided that the │ -│ above copyright notice and this permission notice appear in all copies. │ -│ │ -│ THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL │ -│ WARRANTIES WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED │ -│ WARRANTIES OF MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE │ -│ AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL │ -│ DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR │ -│ PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER │ -│ TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR │ -│ PERFORMANCE OF THIS SOFTWARE. │ -╚─────────────────────────────────────────────────────────────────────────────*/ -#include "libc/intrin/describentoverlapped.internal.h" -#include "libc/intrin/kprintf.h" -#include "libc/macros.internal.h" -#include "libc/nt/struct/overlapped.h" - -const char *(DescribeNtOverlapped)(char b[128], struct NtOverlapped *o) { - int i = 0, n = 128; - bool gotsome = false; - if (!o) return "NULL"; - i += ksnprintf(b + i, MAX(0, n - i), "{"); - - if (o->hEvent) { - if (gotsome) { - i += ksnprintf(b + i, MAX(0, n - i), ", "); - } else { - gotsome = true; - } - i += ksnprintf(b + i, MAX(0, n - i), ".hEvent = %ld", o->hEvent); - } - - if (o->Pointer) { - if (gotsome) { - i += ksnprintf(b + i, MAX(0, n - i), ", "); - } else { - gotsome = true; - } - i += ksnprintf(b + i, MAX(0, n - i), ".Pointer = (void *)%p", o->Pointer); - } - - i += ksnprintf(b + i, MAX(0, n - i), "}"); - return b; -} diff --git a/libc/intrin/describentoverlapped.internal.h b/libc/intrin/describentoverlapped.internal.h deleted file mode 100755 index 8d33a7611..000000000 --- a/libc/intrin/describentoverlapped.internal.h +++ /dev/null @@ -1,13 +0,0 @@ -#ifndef COSMOPOLITAN_LIBC_INTRIN_DESCRIBENTOVERLAPPED_INTERNAL_H_ -#define COSMOPOLITAN_LIBC_INTRIN_DESCRIBENTOVERLAPPED_INTERNAL_H_ -#include "libc/mem/alloca.h" -#include "libc/nt/struct/overlapped.h" -#if !(__ASSEMBLER__ + __LINKER__ + 0) -COSMOPOLITAN_C_START_ - -const char *DescribeNtOverlapped(char[128], struct NtOverlapped *); -#define DescribeNtOverlapped(x) DescribeNtOverlapped(alloca(128), x) - -COSMOPOLITAN_C_END_ -#endif /* !(__ASSEMBLER__ + __LINKER__ + 0) */ -#endif /* COSMOPOLITAN_LIBC_INTRIN_DESCRIBENTOVERLAPPED_INTERNAL_H_ */ diff --git a/libc/intrin/extend.c b/libc/intrin/extend.c deleted file mode 100644 index 3a24354f4..000000000 --- a/libc/intrin/extend.c +++ /dev/null @@ -1,86 +0,0 @@ -/*-*- mode:c;indent-tabs-mode:nil;c-basic-offset:2;tab-width:8;coding:utf-8 -*-│ -│vi: set net ft=c ts=2 sts=2 sw=2 fenc=utf-8 :vi│ -╞══════════════════════════════════════════════════════════════════════════════╡ -│ Copyright 2022 Justine Alexandra Roberts Tunney │ -│ │ -│ Permission to use, copy, modify, and/or distribute this software for │ -│ any purpose with or without fee is hereby granted, provided that the │ -│ above copyright notice and this permission notice appear in all copies. │ -│ │ -│ THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL │ -│ WARRANTIES WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED │ -│ WARRANTIES OF MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE │ -│ AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL │ -│ DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR │ -│ PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER │ -│ TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR │ -│ PERFORMANCE OF THIS SOFTWARE. │ -╚─────────────────────────────────────────────────────────────────────────────*/ -#include "libc/assert.h" -#include "libc/calls/calls.h" -#include "libc/dce.h" -#include "libc/intrin/asan.internal.h" -#include "libc/intrin/asancodes.h" -#include "libc/macros.internal.h" -#include "libc/runtime/directmap.internal.h" -#include "libc/runtime/memtrack.internal.h" -#include "libc/sysv/consts/map.h" -#include "libc/sysv/consts/prot.h" - -#define G FRAMESIZE - -static void _mapframe(void *p) { - int prot, flags; - struct DirectMap dm; - prot = PROT_READ | PROT_WRITE; - flags = MAP_PRIVATE | MAP_ANONYMOUS | MAP_FIXED; - if ((dm = sys_mmap(p, G, prot, flags, -1, 0)).addr != p) { - notpossible; - } - __mmi_lock(); - if (TrackMemoryInterval(&_mmi, (uintptr_t)p >> 16, (uintptr_t)p >> 16, - dm.maphandle, prot, flags, false, false, 0, G)) { - notpossible; - } - __mmi_unlock(); -} - -/** - * Extends static allocation. - * - * This simple fixed allocator has unusual invariants - * - * !(p & 0xffff) && !(((p >> 3) + 0x7fff8000) & 0xffff) - * - * which must be the case when selecting a starting address. - * - * @param p points to start of memory region - * @param n specifies how many bytes are needed - * @param e points to end of memory that's allocated - * @param h is highest address to which `e` may grow - * @return new value for `e` - */ -noasan void *_extend(void *p, size_t n, void *e, intptr_t h) { - char *q; -#ifndef NDEBUG - if ((uintptr_t)SHADOW(p) & (G - 1)) notpossible; - if ((uintptr_t)p + (G << kAsanScale) > h) notpossible; -#endif - for (q = e; q < ((char *)p + n); q += 8) { - if (!((uintptr_t)q & (G - 1))) { - if (q + G > (char *)h) notpossible; - _mapframe(q); - if (IsAsan()) { - if (!((uintptr_t)SHADOW(q) & (G - 1))) { - _mapframe(SHADOW(q)); - __asan_poison(q, G << kAsanScale, kAsanProtected); - } - } - } - if (IsAsan()) { - *SHADOW(q) = 0; - } - } - asm("mfence"); - return q; -} diff --git a/libc/intrin/fds_lock.c b/libc/intrin/fds_lock.c index 6097b4a5a..b01bccb9e 100644 --- a/libc/intrin/fds_lock.c +++ b/libc/intrin/fds_lock.c @@ -17,7 +17,7 @@ │ PERFORMANCE OF THIS SOFTWARE. │ ╚─────────────────────────────────────────────────────────────────────────────*/ #include "libc/calls/state.internal.h" -#include "libc/thread/thread.h" +#include "libc/intrin/pthread.h" void(__fds_lock)(void) { pthread_mutex_lock(&__fds_lock_obj); diff --git a/libc/intrin/ftrace.c b/libc/intrin/ftrace.c index e63f11d68..202fe746c 100644 --- a/libc/intrin/ftrace.c +++ b/libc/intrin/ftrace.c @@ -16,7 +16,6 @@ │ TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR │ │ PERFORMANCE OF THIS SOFTWARE. │ ╚─────────────────────────────────────────────────────────────────────────────*/ -#include "libc/atomic.h" #include "libc/runtime/runtime.h" /** @@ -40,4 +39,4 @@ * though under normal circumstances, `__ftrace` should only be either * zero or one. */ -atomic_int __ftrace; +_Atomic(int) __ftrace; diff --git a/libc/intrin/futex.S b/libc/intrin/futex.S index 1ff350b72..419bf82ba 100644 --- a/libc/intrin/futex.S +++ b/libc/intrin/futex.S @@ -20,14 +20,11 @@ #include "libc/macros.internal.h" .privileged -_futex: push %rbp - mov %rsp,%rbp - mov %rcx,%r10 +_futex: mov %rcx,%r10 mov __NR_futex,%eax clc syscall jnc 1f neg %eax -1: pop %rbp - ret +1: ret .endfn _futex,globl,hidden diff --git a/libc/intrin/futex.internal.h b/libc/intrin/futex.internal.h new file mode 100644 index 000000000..732c10891 --- /dev/null +++ b/libc/intrin/futex.internal.h @@ -0,0 +1,12 @@ +#ifndef COSMOPOLITAN_LIBC_INTRIN_FUTEX_INTERNAL_H_ +#define COSMOPOLITAN_LIBC_INTRIN_FUTEX_INTERNAL_H_ +#include "libc/calls/struct/timespec.h" +#if !(__ASSEMBLER__ + __LINKER__ + 0) +COSMOPOLITAN_C_START_ + +int _futex_wait(void *, int, char, struct timespec *) hidden; +int _futex_wake(void *, int, char) hidden; + +COSMOPOLITAN_C_END_ +#endif /* !(__ASSEMBLER__ + __LINKER__ + 0) */ +#endif /* COSMOPOLITAN_LIBC_INTRIN_FUTEX_INTERNAL_H_ */ diff --git a/libc/intrin/unlockfileex.c b/libc/intrin/futex_wait.c similarity index 63% rename from libc/intrin/unlockfileex.c rename to libc/intrin/futex_wait.c index 8d2908463..bb6a6c5c6 100644 --- a/libc/intrin/unlockfileex.c +++ b/libc/intrin/futex_wait.c @@ -17,29 +17,34 @@ │ PERFORMANCE OF THIS SOFTWARE. │ ╚─────────────────────────────────────────────────────────────────────────────*/ #include "libc/calls/strace.internal.h" -#include "libc/calls/syscall_support-nt.internal.h" -#include "libc/intrin/describentoverlapped.internal.h" -#include "libc/nt/files.h" -#include "libc/nt/struct/overlapped.h" +#include "libc/calls/struct/timespec.h" +#include "libc/calls/struct/timespec.internal.h" +#include "libc/dce.h" +#include "libc/errno.h" +#include "libc/intrin/describeflags.internal.h" +#include "libc/intrin/futex.internal.h" +#include "libc/intrin/pthread.h" +#include "libc/sysv/consts/futex.h" -__msabi extern typeof(UnlockFileEx) *const __imp_UnlockFileEx; +int _futex(void *, int, int, struct timespec *) hidden; -/** - * Unlocks file on the New Technology. - * - * @return handle, or -1 on failure - * @note this wrapper takes care of ABI, STRACE(), and __winerr() - */ -bool32 UnlockFileEx(int64_t hFile, uint32_t dwReserved, - uint32_t nNumberOfBytesToUnlockLow, - uint32_t nNumberOfBytesToUnlockHigh, - struct NtOverlapped *lpOverlapped) { - bool32 ok; - ok = __imp_UnlockFileEx(hFile, dwReserved, nNumberOfBytesToUnlockLow, - nNumberOfBytesToUnlockHigh, lpOverlapped); - if (!ok) __winerr(); - STRACE("UnlockFileEx(%ld, %#x, %'zu, %s) → %hhhd% m", hFile, dwReserved, - (uint64_t)nNumberOfBytesToUnlockHigh << 32 | nNumberOfBytesToUnlockLow, - DescribeNtOverlapped(lpOverlapped), ok); - return ok; +int _futex_wait(void *addr, int expect, char pshared, + struct timespec *timeout) { + int op, ax, pf; + if (IsLinux() || IsOpenbsd()) { + pf = pshared == PTHREAD_PROCESS_PRIVATE ? FUTEX_PRIVATE_FLAG : 0; + op = FUTEX_WAIT | pf; + ax = _futex(addr, op, expect, timeout); + if (SupportsLinux() && pf && ax == -ENOSYS) { + // RHEL5 doesn't support FUTEX_PRIVATE_FLAG + op = FUTEX_WAIT; + ax = _futex(addr, op, expect, timeout); + } + if (IsOpenbsd() && ax > 0) ax = -ax; // yes openbsd does this w/o cf + STRACE("futex(%t, %s, %d, %s) → %s", addr, DescribeFutexOp(op), expect, + DescribeTimespec(0, timeout), DescribeFutexResult(ax)); + return ax; + } else { + return pthread_yield(); + } } diff --git a/libc/intrin/describentlockfileflags.c b/libc/intrin/futex_wake.c similarity index 70% rename from libc/intrin/describentlockfileflags.c rename to libc/intrin/futex_wake.c index d7afc98c6..d53330a57 100644 --- a/libc/intrin/describentlockfileflags.c +++ b/libc/intrin/futex_wake.c @@ -16,16 +16,31 @@ │ TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR │ │ PERFORMANCE OF THIS SOFTWARE. │ ╚─────────────────────────────────────────────────────────────────────────────*/ +#include "libc/calls/strace.internal.h" +#include "libc/dce.h" +#include "libc/errno.h" #include "libc/intrin/describeflags.internal.h" -#include "libc/macros.internal.h" -#include "libc/nt/enum/filelockflags.h" +#include "libc/intrin/futex.internal.h" +#include "libc/intrin/pthread.h" +#include "libc/sysv/consts/futex.h" -static const struct DescribeFlags kNtLockFileFlags[] = { - {kNtLockfileFailImmediately, "FailImmediately"}, // - {kNtLockfileExclusiveLock, "ExclusiveLock"}, // -}; +int _futex(void *, int, int) hidden; -const char *(DescribeNtLockFileFlags)(char buf[64], uint32_t x) { - return DescribeFlags(buf, 64, kNtLockFileFlags, ARRAYLEN(kNtLockFileFlags), - "kNtLockfile", x); +int _futex_wake(void *addr, int count, char pshared) { + int op, ax, pf; + if (IsLinux() || IsOpenbsd()) { + pf = pshared == PTHREAD_PROCESS_PRIVATE ? FUTEX_PRIVATE_FLAG : 0; + op = FUTEX_WAKE | pf; + ax = _futex(addr, op, count); + if (SupportsLinux() && pf && ax == -ENOSYS) { + // RHEL5 doesn't support FUTEX_PRIVATE_FLAG + op = FUTEX_WAKE; + ax = _futex(addr, op, count); + } + STRACE("futex(%t, %s, %d) → %s", addr, DescribeFutexOp(op), count, + DescribeFutexResult(ax)); + return ax; + } else { + return 0; + } } diff --git a/libc/intrin/g_fds.c b/libc/intrin/g_fds.c index f3c0cfe8b..192476725 100644 --- a/libc/intrin/g_fds.c +++ b/libc/intrin/g_fds.c @@ -16,14 +16,13 @@ │ TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR │ │ PERFORMANCE OF THIS SOFTWARE. │ ╚─────────────────────────────────────────────────────────────────────────────*/ -#include "libc/calls/extend.internal.h" #include "libc/calls/internal.h" #include "libc/calls/state.internal.h" +#include "libc/intrin/pthread.h" #include "libc/intrin/pushpop.h" #include "libc/intrin/weaken.h" #include "libc/nt/runtime.h" #include "libc/sysv/consts/o.h" -#include "libc/thread/thread.h" STATIC_YOINK("_init_g_fds"); @@ -34,39 +33,38 @@ static textwindows dontinline void SetupWinStd(struct Fds *fds, int i, int x) { int64_t h; h = GetStdHandle(x); if (!h || h == -1) return; - fds->p[i].kind = pushpop(kFdFile); - fds->p[i].handle = h; + fds->__init_p[i].kind = pushpop(kFdFile); + fds->__init_p[i].handle = h; fds->f = i + 1; } textstartup void InitializeFileDescriptors(void) { struct Fds *fds; - __fds_lock_obj._type = PTHREAD_MUTEX_RECURSIVE; + __fds_lock_obj.type = PTHREAD_MUTEX_RECURSIVE; fds = VEIL("r", &g_fds); - fds->p = fds->e = (void *)0x6fe000040000; - fds->n = 4; + pushmov(&fds->n, ARRAYLEN(fds->__init_p)); fds->f = 3; - fds->e = _extend(fds->p, fds->n * sizeof(*fds->p), fds->e, 0x6ff000000000); + fds->p = fds->__init_p; if (IsMetal()) { extern const char vga_console[]; pushmov(&fds->f, 3ull); if (weaken(vga_console)) { - fds->p[0].kind = pushpop(kFdConsole); - fds->p[1].kind = pushpop(kFdConsole); - fds->p[2].kind = pushpop(kFdConsole); + fds->__init_p[0].kind = pushpop(kFdConsole); + fds->__init_p[1].kind = pushpop(kFdConsole); + fds->__init_p[2].kind = pushpop(kFdConsole); } else { - fds->p[0].kind = pushpop(kFdSerial); - fds->p[1].kind = pushpop(kFdSerial); - fds->p[2].kind = pushpop(kFdSerial); + fds->__init_p[0].kind = pushpop(kFdSerial); + fds->__init_p[1].kind = pushpop(kFdSerial); + fds->__init_p[2].kind = pushpop(kFdSerial); } - fds->p[0].handle = VEIL("r", 0x3F8ull); - fds->p[1].handle = VEIL("r", 0x3F8ull); - fds->p[2].handle = VEIL("r", 0x3F8ull); + fds->__init_p[0].handle = VEIL("r", 0x3F8ull); + fds->__init_p[1].handle = VEIL("r", 0x3F8ull); + fds->__init_p[2].handle = VEIL("r", 0x3F8ull); } else if (IsWindows()) { SetupWinStd(fds, 0, kNtStdInputHandle); SetupWinStd(fds, 1, kNtStdOutputHandle); SetupWinStd(fds, 2, kNtStdErrorHandle); } - fds->p[1].flags = O_WRONLY | O_APPEND; - fds->p[2].flags = O_WRONLY | O_APPEND; + fds->__init_p[1].flags = O_WRONLY | O_APPEND; + fds->__init_p[2].flags = O_WRONLY | O_APPEND; } diff --git a/libc/intrin/gettid.c b/libc/intrin/gettid.c index 8b1018399..77ac11678 100644 --- a/libc/intrin/gettid.c +++ b/libc/intrin/gettid.c @@ -16,10 +16,13 @@ │ TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR │ │ PERFORMANCE OF THIS SOFTWARE. │ ╚─────────────────────────────────────────────────────────────────────────────*/ +#include "libc/assert.h" #include "libc/calls/calls.h" #include "libc/calls/state.internal.h" #include "libc/calls/syscall-sysv.internal.h" -#include "libc/thread/tls.h" +#include "libc/dce.h" +#include "libc/nexgen32e/gettls.h" +#include "libc/nexgen32e/threaded.h" /** * Returns current thread id. @@ -28,6 +31,22 @@ * if this is the main thread. On NetBSD, gettid() for the main thread * is always 1. * + * This function issues a system call. That stops being the case as soon + * as __install_tls() is called. That'll happen automatically, when you + * call clone() and provide the TLS parameter. We assume that when a TLS + * block exists, then + * + * *(int *)(__get_tls() + 0x38) + * + * will contain the thread id. Therefore when issuing clone() calls, the + * `CLONE_CHILD_SETTID` and `CLONE_CHILD_CLEARTID` flags should use that + * index as its `ctid` memory. + * + * gettid (single threaded) l: 126𝑐 41𝑛𝑠 + * gettid (tls enabled) l: 2𝑐 1𝑛𝑠 + * + * The TLS convention is important for reentrant lock performance. + * * @return thread id greater than zero or -1 w/ errno * @asyncsignalsafe * @threadsafe @@ -36,7 +55,7 @@ int gettid(void) { int tid; if (__tls_enabled && !__vforked) { - tid = __get_tls()->tib_tid; + tid = *(int *)(__get_tls() + 0x38); if (tid > 0) { return tid; } diff --git a/libc/intrin/intrin.mk b/libc/intrin/intrin.mk index ddc5615e2..891ae56c0 100644 --- a/libc/intrin/intrin.mk +++ b/libc/intrin/intrin.mk @@ -43,20 +43,6 @@ $(LIBC_INTRIN_A).pkg: \ $(LIBC_INTRIN_A_OBJS) \ $(foreach x,$(LIBC_INTRIN_A_DIRECTDEPS),$($(x)_A).pkg) -# we can't use asan because: -# asan guard pages haven't been allocated yet -o/$(MODE)/libc/intrin/directmap.o \ -o/$(MODE)/libc/intrin/directmap-nt.o: private \ - OVERRIDE_COPTS += \ - -ffreestanding \ - -fno-sanitize=address - -# we want small code size because: -# to keep .text.head under 4096 bytes -o/$(MODE)/libc/intrin/mman.greg.o: private \ - OVERRIDE_COPTS += \ - -Os - # we can't use asan and ubsan because: # this is asan and ubsan o/$(MODE)/libc/intrin/asan.o \ @@ -93,6 +79,11 @@ o/$(MODE)/libc/intrin/futex_wait.o \ o/$(MODE)/libc/intrin/futex_wake.o \ o/$(MODE)/libc/intrin/gettid.greg.o \ o/$(MODE)/libc/intrin/sys_gettid.greg.o \ +o/$(MODE)/libc/intrin/pthread_mutex_lock.o \ +o/$(MODE)/libc/intrin/pthread_mutex_wait.o \ +o/$(MODE)/libc/intrin/pthread_mutex_wake.o \ +o/$(MODE)/libc/intrin/pthread_mutex_unlock.o \ +o/$(MODE)/libc/intrin/pthread_mutex_trylock.o \ o/$(MODE)/libc/intrin/_trylock_debug_4.o \ o/$(MODE)/libc/intrin/_spinlock_debug_4.o: private \ OVERRIDE_CFLAGS += \ diff --git a/libc/intrin/kprintf.greg.c b/libc/intrin/kprintf.greg.c index fd072467d..6d7bbf1ef 100644 --- a/libc/intrin/kprintf.greg.c +++ b/libc/intrin/kprintf.greg.c @@ -24,8 +24,6 @@ #include "libc/errno.h" #include "libc/fmt/divmod10.internal.h" #include "libc/fmt/fmt.h" -#include "libc/intrin/asan.internal.h" -#include "libc/intrin/asancodes.h" #include "libc/intrin/bits.h" #include "libc/intrin/cmpxchg.h" #include "libc/intrin/kprintf.h" @@ -37,7 +35,9 @@ #include "libc/limits.h" #include "libc/log/internal.h" #include "libc/macros.internal.h" +#include "libc/nexgen32e/gettls.h" #include "libc/nexgen32e/rdtsc.h" +#include "libc/nexgen32e/threaded.h" #include "libc/nexgen32e/uart.internal.h" #include "libc/nt/process.h" #include "libc/nt/runtime.h" @@ -52,8 +52,6 @@ #include "libc/str/utf16.h" #include "libc/sysv/consts/nr.h" #include "libc/sysv/consts/prot.h" -#include "libc/thread/tls.h" -#include "libc/thread/tls2.h" extern hidden struct SymbolTable *__symtab; @@ -183,22 +181,19 @@ privileged static void klog(const char *b, size_t n) { : "=a"(rax), "=D"(rdi), "=S"(rsi), "=d"(rdx) : "0"(__NR_write), "1"(2), "2"(b), "3"(n) : "rcx", "r8", "r9", "r10", "r11", "memory", "cc"); - if (rax < 0) { - notpossible; - } } } privileged static size_t kformat(char *b, size_t n, const char *fmt, va_list va) { - int si, y; + int si; wint_t t, u; const char *abet; signed char type; const char *s, *f; unsigned long long x; unsigned i, j, m, rem, sign, hash, cols, prec; - char c, *p, *e, pdot, zero, flip, dang, base, quot, uppr, ansi, z[128]; + char c, *p, *e, pdot, zero, flip, dang, base, quot, uppr, z[128]; if (kistextpointer(b) || kisdangerous(b)) n = 0; if (!kistextpointer(fmt)) fmt = "!!WONTFMT"; p = b; @@ -317,17 +312,7 @@ privileged static size_t kformat(char *b, size_t n, const char *fmt, if (!__tls_enabled) { x = __pid; } else { - x = __get_tls_privileged()->tib_tid; - } - if (!__nocolor && p + 7 <= e) { - *p++ = '\e'; - *p++ = '['; - *p++ = '1'; - *p++ = ';'; - *p++ = '3'; - *p++ = '0' + x % 8; - *p++ = 'm'; - ansi = true; + x = *(int *)(__get_tls_privileged() + 0x38); } } else { x = 666; @@ -705,15 +690,6 @@ privileged static size_t kformat(char *b, size_t n, const char *fmt, } break; } - if (ansi) { - if (p + 4 <= e) { - *p++ = '\e'; - *p++ = '['; - *p++ = '0'; - *p++ = 'm'; - } - ansi = false; - } break; } } diff --git a/libc/intrin/leaky.internal.h b/libc/intrin/leaky.internal.h deleted file mode 100644 index 225671a9a..000000000 --- a/libc/intrin/leaky.internal.h +++ /dev/null @@ -1,16 +0,0 @@ -#ifndef COSMOPOLITAN_LIBC_INTRIN_LEAKY_INTERNAL_H_ -#define COSMOPOLITAN_LIBC_INTRIN_LEAKY_INTERNAL_H_ -#if !(__ASSEMBLER__ + __LINKER__ + 0) -COSMOPOLITAN_C_START_ - -#define IGNORE_LEAKS(FUNC) \ - STATIC_YOINK("_leaky_start"); \ - void *_leaky_##FUNC[] _Section(".piro.relo.sort.leaky.2." #FUNC \ - ",\"aw\",@init_array #") = {FUNC} - -extern intptr_t _leaky_end[] __attribute__((__weak__)); -extern intptr_t _leaky_start[] __attribute__((__weak__)); - -COSMOPOLITAN_C_END_ -#endif /* !(__ASSEMBLER__ + __LINKER__ + 0) */ -#endif /* COSMOPOLITAN_LIBC_INTRIN_LEAKY_INTERNAL_H_ */ diff --git a/libc/intrin/lockcmpxchg.c b/libc/intrin/lockcmpxchg.c new file mode 100644 index 000000000..126659419 --- /dev/null +++ b/libc/intrin/lockcmpxchg.c @@ -0,0 +1,47 @@ +/*-*- mode:c;indent-tabs-mode:nil;c-basic-offset:2;tab-width:8;coding:utf-8 -*-│ +│vi: set net ft=c ts=2 sts=2 sw=2 fenc=utf-8 :vi│ +╞══════════════════════════════════════════════════════════════════════════════╡ +│ Copyright 2021 Justine Alexandra Roberts Tunney │ +│ │ +│ Permission to use, copy, modify, and/or distribute this software for │ +│ any purpose with or without fee is hereby granted, provided that the │ +│ above copyright notice and this permission notice appear in all copies. │ +│ │ +│ THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL │ +│ WARRANTIES WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED │ +│ WARRANTIES OF MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE │ +│ AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL │ +│ DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR │ +│ PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER │ +│ TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR │ +│ PERFORMANCE OF THIS SOFTWARE. │ +╚─────────────────────────────────────────────────────────────────────────────*/ +#include "libc/intrin/lockcmpxchg.h" + +/** + * Compares and exchanges w/ lock prefix. + * + * @param ifthing is uint𝑘_t[hasatleast 1] where 𝑘 ∈ {8,16,32,64} + * @param size is automatically supplied by macro wrapper + * @return true if value was exchanged, otherwise false + * @see cmpxchg() if only written by one thread + */ +bool(_lockcmpxchg)(void *ifthing, intptr_t isequaltome, + intptr_t replaceitwithme, size_t size) { + switch (size) { + case 1: + return _lockcmpxchg((int8_t *)ifthing, (int8_t)isequaltome, + (int8_t)replaceitwithme); + case 2: + return _lockcmpxchg((int16_t *)ifthing, (int16_t)isequaltome, + (int16_t)replaceitwithme); + case 4: + return _lockcmpxchg((int32_t *)ifthing, (int32_t)isequaltome, + (int32_t)replaceitwithme); + case 8: + return _lockcmpxchg((int64_t *)ifthing, (int64_t)isequaltome, + (int64_t)replaceitwithme); + default: + return false; + } +} diff --git a/libc/intrin/lockcmpxchg.h b/libc/intrin/lockcmpxchg.h index d025e8d58..6f8b687ed 100644 --- a/libc/intrin/lockcmpxchg.h +++ b/libc/intrin/lockcmpxchg.h @@ -4,6 +4,8 @@ #if !(__ASSEMBLER__ + __LINKER__ + 0) COSMOPOLITAN_C_START_ +bool _lockcmpxchg(void *, intptr_t, intptr_t, size_t); + #if defined(__GNUC__) && !defined(__STRICT_ANSI__) && defined(__x86__) #define _lockcmpxchg(IFTHING, ISEQUALTOME, REPLACEITWITHME) \ ({ \ @@ -17,6 +19,9 @@ COSMOPOLITAN_C_START_ : "cc"); \ DidIt; \ }) +#else +#define _lockcmpxchg(MEM, CMP, VAL) \ + _lockcmpxchg(MEM, (intptr_t)(CMP), (intptr_t)(VAL), sizeof(*(MEM))) #endif /* GNUC && !ANSI && x86 */ COSMOPOLITAN_C_END_ diff --git a/libc/intrin/lockfileex.c b/libc/intrin/lockfileex.c deleted file mode 100644 index b9d97587c..000000000 --- a/libc/intrin/lockfileex.c +++ /dev/null @@ -1,50 +0,0 @@ -/*-*- mode:c;indent-tabs-mode:nil;c-basic-offset:2;tab-width:8;coding:utf-8 -*-│ -│vi: set net ft=c ts=2 sts=2 sw=2 fenc=utf-8 :vi│ -╞══════════════════════════════════════════════════════════════════════════════╡ -│ Copyright 2022 Justine Alexandra Roberts Tunney │ -│ │ -│ Permission to use, copy, modify, and/or distribute this software for │ -│ any purpose with or without fee is hereby granted, provided that the │ -│ above copyright notice and this permission notice appear in all copies. │ -│ │ -│ THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL │ -│ WARRANTIES WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED │ -│ WARRANTIES OF MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE │ -│ AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL │ -│ DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR │ -│ PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER │ -│ TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR │ -│ PERFORMANCE OF THIS SOFTWARE. │ -╚─────────────────────────────────────────────────────────────────────────────*/ -#include "libc/calls/strace.internal.h" -#include "libc/calls/syscall_support-nt.internal.h" -#include "libc/intrin/describeflags.internal.h" -#include "libc/intrin/describentoverlapped.internal.h" -#include "libc/nt/files.h" - -__msabi extern typeof(LockFileEx) *const __imp_LockFileEx; - -/** - * Locks file on the New Technology. - * - * @return handle, or -1 on failure - * @note this wrapper takes care of ABI, STRACE(), and __winerr() - */ -bool32 LockFileEx(int64_t hFile, uint32_t dwFlags, uint32_t dwReserved, - uint32_t nNumberOfBytesToLockLow, - uint32_t nNumberOfBytesToLockHigh, - struct NtOverlapped *lpOverlapped) { - bool32 ok; - STRACE("LockFileEx(%ld, %s, %#x, %'zu, %s) → ...", hFile, - DescribeNtLockFileFlags(dwFlags), dwReserved, - (uint64_t)nNumberOfBytesToLockHigh << 32 | nNumberOfBytesToLockLow, - DescribeNtOverlapped(lpOverlapped)); - ok = __imp_LockFileEx(hFile, dwFlags, dwReserved, nNumberOfBytesToLockLow, - nNumberOfBytesToLockHigh, lpOverlapped); - if (!ok) __winerr(); - STRACE("LockFileEx(%ld, %s, %#x, %'zu, [%s]) → %hhhd% m", hFile, - DescribeNtLockFileFlags(dwFlags), dwReserved, - (uint64_t)nNumberOfBytesToLockHigh << 32 | nNumberOfBytesToLockLow, - DescribeNtOverlapped(lpOverlapped), ok); - return ok; -} diff --git a/libc/calls/_timespec_cmp.c b/libc/intrin/lockxadd.c similarity index 66% rename from libc/calls/_timespec_cmp.c rename to libc/intrin/lockxadd.c index 46b9bf122..653a6377d 100644 --- a/libc/calls/_timespec_cmp.c +++ b/libc/intrin/lockxadd.c @@ -1,7 +1,7 @@ /*-*- mode:c;indent-tabs-mode:nil;c-basic-offset:2;tab-width:8;coding:utf-8 -*-│ │vi: set net ft=c ts=2 sts=2 sw=2 fenc=utf-8 :vi│ ╞══════════════════════════════════════════════════════════════════════════════╡ -│ Copyright 2022 Justine Alexandra Roberts Tunney │ +│ Copyright 2021 Justine Alexandra Roberts Tunney │ │ │ │ Permission to use, copy, modify, and/or distribute this software for │ │ any purpose with or without fee is hereby granted, provided that the │ @@ -16,15 +16,29 @@ │ TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR │ │ PERFORMANCE OF THIS SOFTWARE. │ ╚─────────────────────────────────────────────────────────────────────────────*/ -#include "libc/calls/struct/timespec.h" +#include "libc/intrin/lockxadd.h" +#include "libc/runtime/runtime.h" /** - * Compares two nanosecond timestamps. + * Compares and exchanges w/ lock prefix. + * + * @param ifthing is uint𝑘_t[hasatleast 1] where 𝑘 ∈ {8,16,32,64} + * @param size is automatically supplied by macro wrapper + * @return value at location `*ifthing` *before* addition + * @see InterlockedAdd() for a very similar API + * @see xadd() if only written by one thread */ -int _timespec_cmp(struct timespec a, struct timespec b) { - int cmp; - if (!(cmp = (a.tv_sec > b.tv_sec) - (a.tv_sec < b.tv_sec))) { - cmp = (a.tv_nsec > b.tv_nsec) - (a.tv_nsec < b.tv_nsec); +intptr_t(_lockxadd)(void *ifthing, intptr_t replaceitwithme, size_t size) { + switch (size) { + case 1: + return _lockxadd((int8_t *)ifthing, (int8_t)replaceitwithme); + case 2: + return _lockxadd((int16_t *)ifthing, (int16_t)replaceitwithme); + case 4: + return _lockxadd((int32_t *)ifthing, (int32_t)replaceitwithme); + case 8: + return _lockxadd((int64_t *)ifthing, (int64_t)replaceitwithme); + default: + abort(); } - return cmp; } diff --git a/libc/thread/pthread_rwlock_unlock.c b/libc/intrin/lockxchg.c similarity index 70% rename from libc/thread/pthread_rwlock_unlock.c rename to libc/intrin/lockxchg.c index 617746146..a372ff997 100644 --- a/libc/thread/pthread_rwlock_unlock.c +++ b/libc/intrin/lockxchg.c @@ -1,7 +1,7 @@ /*-*- mode:c;indent-tabs-mode:nil;c-basic-offset:2;tab-width:8;coding:utf-8 -*-│ │vi: set net ft=c ts=2 sts=2 sw=2 fenc=utf-8 :vi│ ╞══════════════════════════════════════════════════════════════════════════════╡ -│ Copyright 2022 Justine Alexandra Roberts Tunney │ +│ Copyright 2021 Justine Alexandra Roberts Tunney │ │ │ │ Permission to use, copy, modify, and/or distribute this software for │ │ any purpose with or without fee is hereby granted, provided that the │ @@ -16,21 +16,27 @@ │ TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR │ │ PERFORMANCE OF THIS SOFTWARE. │ ╚─────────────────────────────────────────────────────────────────────────────*/ -#include "libc/thread/thread.h" -#include "third_party/nsync/mu.h" +#include "libc/intrin/lockxchg.h" /** - * Unlocks read-write lock. + * Compares and exchanges w/ lock prefix. * - * @return 0 on success, or errno on error - * @raise EINVAL if lock is in a bad state + * @param memory is uint𝑘_t[hasatleast 1] where 𝑘 ∈ {8,16,32,64} + * @param size is automatically supplied by macro wrapper + * @return true if value was exchanged, otherwise false + * @see xchg() */ -int pthread_rwlock_unlock(pthread_rwlock_t *rwlock) { - if (rwlock->_iswrite) { - rwlock->_iswrite = 0; - nsync_mu_unlock((nsync_mu *)rwlock); - } else { - nsync_mu_runlock((nsync_mu *)rwlock); +intptr_t(lockxchg)(void *memory, void *localvar, size_t size) { + switch (size) { + case 1: + return lockxchg((int8_t *)memory, (int8_t *)localvar); + case 2: + return lockxchg((int16_t *)memory, (int16_t *)localvar); + case 4: + return lockxchg((int32_t *)memory, (int32_t *)localvar); + case 8: + return lockxchg((int64_t *)memory, (int64_t *)localvar); + default: + return false; } - return 0; } diff --git a/libc/intrin/mmi.init.S b/libc/intrin/mmi.init.S index 5910d3c0f..54842d9c8 100644 --- a/libc/intrin/mmi.init.S +++ b/libc/intrin/mmi.init.S @@ -16,11 +16,11 @@ │ TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR │ │ PERFORMANCE OF THIS SOFTWARE. │ ╚─────────────────────────────────────────────────────────────────────────────*/ -#include "libc/thread/thread.h" +#include "libc/intrin/pthread.h" #include "libc/macros.internal.h" .init.start 200,_init__mmi movb $OPEN_MAX,_mmi+8 movl $_mmi+24,_mmi+16 - movb $PTHREAD_MUTEX_RECURSIVE,__mmi_lock_obj+16(%rip) + movb $PTHREAD_MUTEX_RECURSIVE,__mmi_lock_obj(%rip) .init.end 200,_init__mmi diff --git a/libc/intrin/mmi_lock.c b/libc/intrin/mmi_lock.c index 0bcbed171..7dc44fd97 100644 --- a/libc/intrin/mmi_lock.c +++ b/libc/intrin/mmi_lock.c @@ -16,10 +16,8 @@ │ TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR │ │ PERFORMANCE OF THIS SOFTWARE. │ ╚─────────────────────────────────────────────────────────────────────────────*/ +#include "libc/intrin/pthread.h" #include "libc/runtime/memtrack.internal.h" -#include "libc/thread/thread.h" - -// this lock currently needs to be (1) recursive and (2) not nsync extern pthread_mutex_t __mmi_lock_obj; diff --git a/libc/intrin/mmi_lock_obj.c b/libc/intrin/mmi_lock_obj.c index e33861b9d..c34d116bc 100644 --- a/libc/intrin/mmi_lock_obj.c +++ b/libc/intrin/mmi_lock_obj.c @@ -16,6 +16,6 @@ │ TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR │ │ PERFORMANCE OF THIS SOFTWARE. │ ╚─────────────────────────────────────────────────────────────────────────────*/ -#include "libc/thread/thread.h" +#include "libc/intrin/pthread.h" pthread_mutex_t __mmi_lock_obj; // recursive :'( diff --git a/libc/intrin/nopl.internal.h b/libc/intrin/nopl.h similarity index 100% rename from libc/intrin/nopl.internal.h rename to libc/intrin/nopl.h diff --git a/libc/thread/pthread_attr_setschedpolicy.c b/libc/intrin/phtread_attr_setschedpolicy.c similarity index 98% rename from libc/thread/pthread_attr_setschedpolicy.c rename to libc/intrin/phtread_attr_setschedpolicy.c index dafa1f46c..4de984b9d 100644 --- a/libc/thread/pthread_attr_setschedpolicy.c +++ b/libc/intrin/phtread_attr_setschedpolicy.c @@ -16,7 +16,7 @@ │ TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR │ │ PERFORMANCE OF THIS SOFTWARE. │ ╚─────────────────────────────────────────────────────────────────────────────*/ -#include "libc/thread/thread.h" +#include "libc/intrin/pthread.h" /** * Sets thread scheduler policy attribute, e.g. diff --git a/libc/thread/pthread_cancel.c b/libc/intrin/pthread.c similarity index 92% rename from libc/thread/pthread_cancel.c rename to libc/intrin/pthread.c index 876589fe1..0eed78739 100644 --- a/libc/thread/pthread_cancel.c +++ b/libc/intrin/pthread.c @@ -1,7 +1,7 @@ /*-*- mode:c;indent-tabs-mode:nil;c-basic-offset:2;tab-width:8;coding:utf-8 -*-│ │vi: set net ft=c ts=2 sts=2 sw=2 fenc=utf-8 :vi│ ╞══════════════════════════════════════════════════════════════════════════════╡ -│ Copyright 2022 Justine Alexandra Roberts Tunney │ +│ Copyright 2021 Justine Alexandra Roberts Tunney │ │ │ │ Permission to use, copy, modify, and/or distribute this software for │ │ any purpose with or without fee is hereby granted, provided that the │ @@ -17,8 +17,12 @@ │ PERFORMANCE OF THIS SOFTWARE. │ ╚─────────────────────────────────────────────────────────────────────────────*/ #include "libc/errno.h" -#include "libc/thread/thread.h" +#include "libc/intrin/pthread.h" int pthread_cancel(pthread_t thread) { return ESRCH; } + +void *__tls_get_addr(size_t v[2]) { + return NULL; +} diff --git a/libc/intrin/pthread.h b/libc/intrin/pthread.h new file mode 100644 index 000000000..576beb427 --- /dev/null +++ b/libc/intrin/pthread.h @@ -0,0 +1,205 @@ +#ifndef COSMOPOLITAN_LIBC_RUNTIME_PTHREAD_H_ +#define COSMOPOLITAN_LIBC_RUNTIME_PTHREAD_H_ + +#define PTHREAD_KEYS_MAX 64 +#define PTHREAD_STACK_MIN FRAMESIZE +#define PTHREAD_DESTRUCTOR_ITERATIONS 4 + +#define PTHREAD_BARRIER_SERIAL_THREAD 31337 + +#define PTHREAD_MUTEX_DEFAULT PTHREAD_MUTEX_NORMAL +#define PTHREAD_MUTEX_NORMAL 0 +#define PTHREAD_MUTEX_RECURSIVE 1 +#define PTHREAD_MUTEX_ERRORCHECK 2 +#define PTHREAD_MUTEX_STALLED 0 +#define PTHREAD_MUTEX_ROBUST 1 + +#define PTHREAD_PROCESS_DEFAULT PTHREAD_PROCESS_PRIVATE +#define PTHREAD_PROCESS_PRIVATE 0 +#define PTHREAD_PROCESS_SHARED 1 + +#define PTHREAD_CREATE_JOINABLE 0 +#define PTHREAD_CREATE_DETACHED 1 + +#define PTHREAD_INHERIT_SCHED 0 +#define PTHREAD_EXPLICIT_SCHED 1 + +#if !(__ASSEMBLER__ + __LINKER__ + 0) +COSMOPOLITAN_C_START_ + +/* clang-format off */ +#define PTHREAD_ONCE_INIT {0} +#define PTHREAD_COND_INITIALIZER {PTHREAD_PROCESS_DEFAULT} +#define PTHREAD_BARRIER_INITIALIZER {PTHREAD_PROCESS_DEFAULT} +#define PTHREAD_RWLOCK_INITIALIZER {PTHREAD_PROCESS_DEFAULT} +#define PTHREAD_MUTEX_INITIALIZER {PTHREAD_MUTEX_DEFAULT, \ + PTHREAD_PROCESS_DEFAULT} +/* clang-format on */ + +typedef uintptr_t pthread_t; +typedef int pthread_id_np_t; +typedef char pthread_condattr_t; +typedef char pthread_rwlockattr_t; +typedef char pthread_barrierattr_t; +typedef unsigned pthread_key_t; +typedef void (*pthread_key_dtor)(void *); + +typedef struct pthread_once_s { + _Atomic(char) lock; +} pthread_once_t; + +typedef struct pthread_spinlock_s { + _Atomic(char) lock; +} pthread_spinlock_t; + +typedef struct pthread_mutex_s { + char type; + char pshared; + int reent; + _Atomic(int) lock; + _Atomic(int) waits; +} pthread_mutex_t; + +typedef struct pthread_mutexattr_s { + char type; + char pshared; +} pthread_mutexattr_t; + +typedef struct pthread_cond_s { + char pshared; + _Atomic(int) waits; + _Atomic(unsigned) seq; +} pthread_cond_t; + +typedef struct pthread_barrier_s { + char pshared; + int count; + _Atomic(int) waits; + _Atomic(int) popped; +} pthread_barrier_t; + +typedef struct pthread_rwlock_s { + char pshared; + _Atomic(int) lock; + _Atomic(int) waits; +} pthread_rwlock_t; + +typedef struct pthread_attr_s { + char detachstate; + char inheritsched; + int schedparam; + int schedpolicy; + int scope; + unsigned guardsize; + unsigned stacksize; + char *stackaddr; +} pthread_attr_t; + +int pthread_yield(void); +void pthread_exit(void *) wontreturn; +pthread_t pthread_self(void) pureconst; +pthread_id_np_t pthread_getthreadid_np(void); +int64_t pthread_getunique_np(pthread_t); +int pthread_setname_np(pthread_t, const char *); +int pthread_getname_np(pthread_t, char *, size_t); +int pthread_getattr_np(pthread_t, pthread_attr_t *); +int pthread_attr_init(pthread_attr_t *); +int pthread_attr_destroy(pthread_attr_t *); +int pthread_attr_getdetachstate(const pthread_attr_t *, int *); +int pthread_attr_setdetachstate(pthread_attr_t *, int); +int pthread_attr_getguardsize(const pthread_attr_t *, size_t *); +int pthread_attr_setguardsize(pthread_attr_t *, size_t); +int pthread_attr_getinheritsched(const pthread_attr_t *, int *); +int pthread_attr_setinheritsched(pthread_attr_t *, int); +int pthread_attr_getschedpolicy(const pthread_attr_t *, int *); +int pthread_attr_setschedpolicy(pthread_attr_t *, int); +int pthread_attr_getscope(const pthread_attr_t *, int *); +int pthread_attr_setscope(pthread_attr_t *, int); +int pthread_attr_getstack(const pthread_attr_t *, void **, size_t *); +int pthread_attr_setstack(pthread_attr_t *, void *, size_t); +int pthread_attr_getstacksize(const pthread_attr_t *, size_t *); +int pthread_attr_setstacksize(pthread_attr_t *, size_t); +int pthread_create(pthread_t *, const pthread_attr_t *, void *(*)(void *), + void *); +int pthread_detach(pthread_t); +int pthread_cancel(pthread_t); +int pthread_join(pthread_t, void **); +int pthread_equal(pthread_t, pthread_t); +int pthread_once(pthread_once_t *, void (*)(void)); +int pthread_spin_init(pthread_spinlock_t *, int); +int pthread_spin_destroy(pthread_spinlock_t *); +int pthread_spin_lock(pthread_spinlock_t *); +int pthread_spin_unlock(pthread_spinlock_t *); +int pthread_spin_trylock(pthread_spinlock_t *); +int pthread_mutexattr_init(pthread_mutexattr_t *); +int pthread_mutexattr_destroy(pthread_mutexattr_t *); +int pthread_mutexattr_gettype(const pthread_mutexattr_t *, int *); +int pthread_mutexattr_settype(pthread_mutexattr_t *, int); +int pthread_mutexattr_setpshared(pthread_mutexattr_t *, int); +int pthread_mutexattr_getpshared(const pthread_mutexattr_t *, int *); +int pthread_mutex_init(pthread_mutex_t *, const pthread_mutexattr_t *); +int pthread_mutex_lock(pthread_mutex_t *); +int pthread_mutex_unlock(pthread_mutex_t *); +int pthread_mutex_trylock(pthread_mutex_t *); +int pthread_mutex_destroy(pthread_mutex_t *); +int pthread_mutex_consistent(pthread_mutex_t *); +int pthread_condattr_init(pthread_condattr_t *); +int pthread_condattr_destroy(pthread_condattr_t *); +int pthread_condattr_setpshared(pthread_condattr_t *, int); +int pthread_condattr_getpshared(const pthread_condattr_t *, int *); +int pthread_cond_init(pthread_cond_t *, const pthread_condattr_t *); +int pthread_cond_destroy(pthread_cond_t *); +int pthread_cond_wait(pthread_cond_t *, pthread_mutex_t *); +int pthread_cond_broadcast(pthread_cond_t *); +int pthread_cond_signal(pthread_cond_t *); +int pthread_rwlockattr_init(pthread_rwlockattr_t *); +int pthread_rwlockattr_destroy(pthread_rwlockattr_t *); +int pthread_rwlockattr_setpshared(pthread_rwlockattr_t *, int); +int pthread_rwlockattr_getpshared(const pthread_rwlockattr_t *, int *); +int pthread_rwlock_init(pthread_rwlock_t *, const pthread_rwlockattr_t *); +int pthread_rwlock_destroy(pthread_rwlock_t *); +int pthread_rwlock_rdlock(pthread_rwlock_t *); +int pthread_rwlock_tryrdlock(pthread_rwlock_t *); +int pthread_rwlock_wrlock(pthread_rwlock_t *); +int pthread_rwlock_trywrlock(pthread_rwlock_t *); +int pthread_rwlock_unlock(pthread_rwlock_t *); +int pthread_key_create(pthread_key_t *, pthread_key_dtor); +int pthread_key_delete(pthread_key_t); +int pthread_setspecific(pthread_key_t, void *); +void *pthread_getspecific(pthread_key_t); +int pthread_barrierattr_init(pthread_barrierattr_t *); +int pthread_barrierattr_destroy(pthread_barrierattr_t *); +int pthread_barrierattr_getpshared(const pthread_barrierattr_t *, int *); +int pthread_barrierattr_setpshared(pthread_barrierattr_t *, int); +int pthread_barrier_wait(pthread_barrier_t *); +int pthread_barrier_destroy(pthread_barrier_t *); +int pthread_barrier_init(pthread_barrier_t *, const pthread_barrierattr_t *, + unsigned); + +#define pthread_spin_init(pSpin, multiprocess) ((pSpin)->lock = 0, 0) +#define pthread_spin_destroy(pSpin) ((pSpin)->lock = -1, 0) +#if (__GNUC__ + 0) * 100 + (__GNUC_MINOR__ + 0) >= 407 && \ + !defined(__STRICT_ANSI__) +extern const errno_t EBUSY; +#define pthread_spin_lock(pSpin) \ + ({ \ + pthread_spinlock_t *_s = pSpin; \ + while (__atomic_test_and_set(&_s->lock, __ATOMIC_SEQ_CST)) donothing; \ + 0; \ + }) +#define pthread_spin_unlock(pSpin) \ + ({ \ + pthread_spinlock_t *_s = pSpin; \ + __atomic_store_n(&_s->lock, 0, __ATOMIC_RELAXED); \ + 0; \ + }) +#define pthread_spin_trylock(pSpin) \ + ({ \ + pthread_spinlock_t *_s = pSpin; \ + __atomic_test_and_set(&_s->lock, __ATOMIC_SEQ_CST) ? EBUSY : 0; \ + }) +#endif /* GCC 4.7+ */ + +COSMOPOLITAN_C_END_ +#endif /* !(__ASSEMBLER__ + __LINKER__ + 0) */ +#endif /* COSMOPOLITAN_LIBC_RUNTIME_PTHREAD_H_ */ diff --git a/libc/thread/thread2.h b/libc/intrin/pthread2.h similarity index 96% rename from libc/thread/thread2.h rename to libc/intrin/pthread2.h index ec6b4a216..0dffe6892 100644 --- a/libc/thread/thread2.h +++ b/libc/intrin/pthread2.h @@ -3,8 +3,8 @@ #include "libc/calls/struct/cpuset.h" #include "libc/calls/struct/sched_param.h" #include "libc/calls/struct/timespec.h" +#include "libc/intrin/pthread.h" #include "libc/runtime/stack.h" -#include "libc/thread/thread.h" #if !(__ASSEMBLER__ + __LINKER__ + 0) COSMOPOLITAN_C_START_ diff --git a/libc/thread/pthread_attr_destroy.c b/libc/intrin/pthread_attr_destroy.c similarity index 98% rename from libc/thread/pthread_attr_destroy.c rename to libc/intrin/pthread_attr_destroy.c index d93ffa7d2..b8690c31f 100644 --- a/libc/thread/pthread_attr_destroy.c +++ b/libc/intrin/pthread_attr_destroy.c @@ -16,7 +16,7 @@ │ TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR │ │ PERFORMANCE OF THIS SOFTWARE. │ ╚─────────────────────────────────────────────────────────────────────────────*/ -#include "libc/thread/thread.h" +#include "libc/intrin/pthread.h" #include "libc/str/str.h" /** diff --git a/libc/thread/pthread_attr_getdetachstate.c b/libc/intrin/pthread_attr_getdetachstate.c similarity index 98% rename from libc/thread/pthread_attr_getdetachstate.c rename to libc/intrin/pthread_attr_getdetachstate.c index 742d26b00..ada3daf78 100644 --- a/libc/thread/pthread_attr_getdetachstate.c +++ b/libc/intrin/pthread_attr_getdetachstate.c @@ -16,7 +16,7 @@ │ TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR │ │ PERFORMANCE OF THIS SOFTWARE. │ ╚─────────────────────────────────────────────────────────────────────────────*/ -#include "libc/thread/thread.h" +#include "libc/intrin/pthread.h" /** * Gets thread detachable attribute. diff --git a/libc/thread/pthread_attr_getguardsize.c b/libc/intrin/pthread_attr_getguardsize.c similarity index 98% rename from libc/thread/pthread_attr_getguardsize.c rename to libc/intrin/pthread_attr_getguardsize.c index 6b82d638f..b91416e39 100644 --- a/libc/thread/pthread_attr_getguardsize.c +++ b/libc/intrin/pthread_attr_getguardsize.c @@ -16,7 +16,7 @@ │ TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR │ │ PERFORMANCE OF THIS SOFTWARE. │ ╚─────────────────────────────────────────────────────────────────────────────*/ -#include "libc/thread/thread.h" +#include "libc/intrin/pthread.h" /** * Returns size of unmapped pages at bottom of stack. diff --git a/libc/thread/pthread_attr_getinheritsched.c b/libc/intrin/pthread_attr_getinheritsched.c similarity index 98% rename from libc/thread/pthread_attr_getinheritsched.c rename to libc/intrin/pthread_attr_getinheritsched.c index f8b07caa1..1ec8dfaa4 100644 --- a/libc/thread/pthread_attr_getinheritsched.c +++ b/libc/intrin/pthread_attr_getinheritsched.c @@ -16,7 +16,7 @@ │ TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR │ │ PERFORMANCE OF THIS SOFTWARE. │ ╚─────────────────────────────────────────────────────────────────────────────*/ -#include "libc/thread/thread.h" +#include "libc/intrin/pthread.h" /** * Returns thread inherit schedule attribute. diff --git a/libc/thread/pthread_attr_getschedparam.c b/libc/intrin/pthread_attr_getschedparam.c similarity index 98% rename from libc/thread/pthread_attr_getschedparam.c rename to libc/intrin/pthread_attr_getschedparam.c index 25e8634b6..b6170a6da 100644 --- a/libc/thread/pthread_attr_getschedparam.c +++ b/libc/intrin/pthread_attr_getschedparam.c @@ -16,7 +16,7 @@ │ TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR │ │ PERFORMANCE OF THIS SOFTWARE. │ ╚─────────────────────────────────────────────────────────────────────────────*/ -#include "libc/thread/thread2.h" +#include "libc/intrin/pthread2.h" /** * Gets thread scheduler parameter attribute. diff --git a/libc/thread/pthread_attr_getschedpolicy.c b/libc/intrin/pthread_attr_getschedpolicy.c similarity index 98% rename from libc/thread/pthread_attr_getschedpolicy.c rename to libc/intrin/pthread_attr_getschedpolicy.c index 88f8e277a..a19325f93 100644 --- a/libc/thread/pthread_attr_getschedpolicy.c +++ b/libc/intrin/pthread_attr_getschedpolicy.c @@ -16,7 +16,7 @@ │ TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR │ │ PERFORMANCE OF THIS SOFTWARE. │ ╚─────────────────────────────────────────────────────────────────────────────*/ -#include "libc/thread/thread.h" +#include "libc/intrin/pthread.h" /** * Gets thread scheduler policy attribute diff --git a/libc/thread/pthread_attr_getscope.c b/libc/intrin/pthread_attr_getscope.c similarity index 98% rename from libc/thread/pthread_attr_getscope.c rename to libc/intrin/pthread_attr_getscope.c index 3759ae815..fa24f5432 100644 --- a/libc/thread/pthread_attr_getscope.c +++ b/libc/intrin/pthread_attr_getscope.c @@ -16,7 +16,7 @@ │ TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR │ │ PERFORMANCE OF THIS SOFTWARE. │ ╚─────────────────────────────────────────────────────────────────────────────*/ -#include "libc/thread/thread.h" +#include "libc/intrin/pthread.h" int pthread_attr_getscope(const pthread_attr_t *a, int *x) { *x = a->scope; diff --git a/libc/thread/pthread_attr_getstack.c b/libc/intrin/pthread_attr_getstack.c similarity index 98% rename from libc/thread/pthread_attr_getstack.c rename to libc/intrin/pthread_attr_getstack.c index c569c7d65..bed6b5c46 100644 --- a/libc/thread/pthread_attr_getstack.c +++ b/libc/intrin/pthread_attr_getstack.c @@ -16,7 +16,7 @@ │ TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR │ │ PERFORMANCE OF THIS SOFTWARE. │ ╚─────────────────────────────────────────────────────────────────────────────*/ -#include "libc/thread/thread.h" +#include "libc/intrin/pthread.h" #include "libc/runtime/stack.h" /** diff --git a/libc/thread/pthread_attr_getstacksize.c b/libc/intrin/pthread_attr_getstacksize.c similarity index 98% rename from libc/thread/pthread_attr_getstacksize.c rename to libc/intrin/pthread_attr_getstacksize.c index 746ed4f7a..d4c5489b1 100644 --- a/libc/thread/pthread_attr_getstacksize.c +++ b/libc/intrin/pthread_attr_getstacksize.c @@ -16,7 +16,7 @@ │ TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR │ │ PERFORMANCE OF THIS SOFTWARE. │ ╚─────────────────────────────────────────────────────────────────────────────*/ -#include "libc/thread/thread.h" +#include "libc/intrin/pthread.h" #include "libc/runtime/stack.h" /** diff --git a/libc/thread/pthread_attr_init.c b/libc/intrin/pthread_attr_init.c similarity index 98% rename from libc/thread/pthread_attr_init.c rename to libc/intrin/pthread_attr_init.c index b12dfea81..ac6f31166 100644 --- a/libc/thread/pthread_attr_init.c +++ b/libc/intrin/pthread_attr_init.c @@ -16,7 +16,7 @@ │ TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR │ │ PERFORMANCE OF THIS SOFTWARE. │ ╚─────────────────────────────────────────────────────────────────────────────*/ -#include "libc/thread/thread.h" +#include "libc/intrin/pthread.h" #include "libc/runtime/stack.h" /** diff --git a/libc/thread/pthread_attr_setdetachstate.c b/libc/intrin/pthread_attr_setdetachstate.c similarity index 98% rename from libc/thread/pthread_attr_setdetachstate.c rename to libc/intrin/pthread_attr_setdetachstate.c index 22e03d62c..5d37579a5 100644 --- a/libc/thread/pthread_attr_setdetachstate.c +++ b/libc/intrin/pthread_attr_setdetachstate.c @@ -17,7 +17,7 @@ │ PERFORMANCE OF THIS SOFTWARE. │ ╚─────────────────────────────────────────────────────────────────────────────*/ #include "libc/errno.h" -#include "libc/thread/thread.h" +#include "libc/intrin/pthread.h" /** * Sets thread detachable attribute, e.g. diff --git a/libc/thread/pthread_attr_setguardsize.c b/libc/intrin/pthread_attr_setguardsize.c similarity index 98% rename from libc/thread/pthread_attr_setguardsize.c rename to libc/intrin/pthread_attr_setguardsize.c index 40b6a77b8..27f5077e5 100644 --- a/libc/thread/pthread_attr_setguardsize.c +++ b/libc/intrin/pthread_attr_setguardsize.c @@ -16,7 +16,7 @@ │ TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR │ │ PERFORMANCE OF THIS SOFTWARE. │ ╚─────────────────────────────────────────────────────────────────────────────*/ -#include "libc/thread/thread.h" +#include "libc/intrin/pthread.h" /** * Sets size of unmapped pages at bottom of stack. diff --git a/libc/thread/pthread_attr_setinheritsched.c b/libc/intrin/pthread_attr_setinheritsched.c similarity index 98% rename from libc/thread/pthread_attr_setinheritsched.c rename to libc/intrin/pthread_attr_setinheritsched.c index eb2737720..74d6fc352 100644 --- a/libc/thread/pthread_attr_setinheritsched.c +++ b/libc/intrin/pthread_attr_setinheritsched.c @@ -18,7 +18,7 @@ ╚─────────────────────────────────────────────────────────────────────────────*/ #include "libc/assert.h" #include "libc/errno.h" -#include "libc/thread/thread.h" +#include "libc/intrin/pthread.h" /** * Sets thread scheduler inheritance attribute, e.g. diff --git a/libc/thread/pthread_attr_setschedparam.c b/libc/intrin/pthread_attr_setschedparam.c similarity index 98% rename from libc/thread/pthread_attr_setschedparam.c rename to libc/intrin/pthread_attr_setschedparam.c index afcf4e381..70a348d58 100644 --- a/libc/thread/pthread_attr_setschedparam.c +++ b/libc/intrin/pthread_attr_setschedparam.c @@ -18,7 +18,7 @@ ╚─────────────────────────────────────────────────────────────────────────────*/ #include "libc/calls/struct/sched_param.h" #include "libc/errno.h" -#include "libc/thread/thread2.h" +#include "libc/intrin/pthread2.h" /** * Sets thread scheduler parameter attribute, e.g. diff --git a/libc/thread/pthread_attr_setscope.c b/libc/intrin/pthread_attr_setscope.c similarity index 98% rename from libc/thread/pthread_attr_setscope.c rename to libc/intrin/pthread_attr_setscope.c index 89ad4eaa8..365a40120 100644 --- a/libc/thread/pthread_attr_setscope.c +++ b/libc/intrin/pthread_attr_setscope.c @@ -16,7 +16,7 @@ │ TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR │ │ PERFORMANCE OF THIS SOFTWARE. │ ╚─────────────────────────────────────────────────────────────────────────────*/ -#include "libc/thread/thread.h" +#include "libc/intrin/pthread.h" int pthread_attr_setscope(pthread_attr_t *a, int x) { a->scope = x; diff --git a/libc/thread/pthread_attr_setstack.c b/libc/intrin/pthread_attr_setstack.c similarity index 99% rename from libc/thread/pthread_attr_setstack.c rename to libc/intrin/pthread_attr_setstack.c index 93aec84a0..6994b97d8 100644 --- a/libc/thread/pthread_attr_setstack.c +++ b/libc/intrin/pthread_attr_setstack.c @@ -19,7 +19,7 @@ #include "libc/dce.h" #include "libc/errno.h" #include "libc/intrin/asan.internal.h" -#include "libc/thread/thread.h" +#include "libc/intrin/pthread.h" /** * Configures custom allocated stack for thread, e.g. diff --git a/libc/thread/pthread_attr_setstacksize.c b/libc/intrin/pthread_attr_setstacksize.c similarity index 98% rename from libc/thread/pthread_attr_setstacksize.c rename to libc/intrin/pthread_attr_setstacksize.c index 353d708ab..893f68714 100644 --- a/libc/thread/pthread_attr_setstacksize.c +++ b/libc/intrin/pthread_attr_setstacksize.c @@ -17,7 +17,7 @@ │ PERFORMANCE OF THIS SOFTWARE. │ ╚─────────────────────────────────────────────────────────────────────────────*/ #include "libc/errno.h" -#include "libc/thread/thread.h" +#include "libc/intrin/pthread.h" /** * Defines minimum stack size for thread. diff --git a/libc/thread/pthread_barrier_destroy.c b/libc/intrin/pthread_barrier_destroy.c similarity index 90% rename from libc/thread/pthread_barrier_destroy.c rename to libc/intrin/pthread_barrier_destroy.c index c249bbe60..9dca7a62a 100644 --- a/libc/thread/pthread_barrier_destroy.c +++ b/libc/intrin/pthread_barrier_destroy.c @@ -16,9 +16,10 @@ │ TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR │ │ PERFORMANCE OF THIS SOFTWARE. │ ╚─────────────────────────────────────────────────────────────────────────────*/ +#include "libc/assert.h" +#include "libc/errno.h" +#include "libc/intrin/pthread.h" #include "libc/str/str.h" -#include "libc/thread/thread.h" -#include "third_party/nsync/counter.h" /** * Destroys barrier. @@ -27,9 +28,10 @@ * @raise EINVAL if threads are still inside the barrier */ int pthread_barrier_destroy(pthread_barrier_t *barrier) { - if (barrier->_nsync) { - nsync_counter_free(barrier->_nsync); - barrier->_nsync = 0; + if (barrier->waits || barrier->popped) { + assert(!"deadlock"); + return EINVAL; } + memset(barrier, -1, sizeof(*barrier)); return 0; } diff --git a/libc/thread/pthread_barrier_init.c b/libc/intrin/pthread_barrier_init.c similarity index 86% rename from libc/thread/pthread_barrier_init.c rename to libc/intrin/pthread_barrier_init.c index b83a35230..f6ea185b8 100644 --- a/libc/thread/pthread_barrier_init.c +++ b/libc/intrin/pthread_barrier_init.c @@ -16,9 +16,10 @@ │ TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR │ │ PERFORMANCE OF THIS SOFTWARE. │ ╚─────────────────────────────────────────────────────────────────────────────*/ +#include "libc/assert.h" #include "libc/errno.h" -#include "libc/thread/thread.h" -#include "third_party/nsync/counter.h" +#include "libc/intrin/pthread.h" +#include "libc/limits.h" /** * Initializes barrier. @@ -27,14 +28,18 @@ * @param count is how many threads need to call pthread_barrier_wait() * before the barrier is released, which must be greater than zero * @return 0 on success, or error number on failure - * @raise EINVAL if `count` isn't greater than zero - * @raise ENOMEM if insufficient memory exists + * @raise EINVAL if `count` isn't greater than zero or overflows */ int pthread_barrier_init(pthread_barrier_t *barrier, const pthread_barrierattr_t *attr, unsigned count) { - nsync_counter c; - if (!count) return EINVAL; - if (!(c = nsync_counter_new(count))) return ENOMEM; - *barrier = (pthread_barrier_t){._nsync = c}; - return 0; + if (count && count < INT_MAX / 2) { + *barrier = (pthread_barrier_t){ + attr ? *attr : PTHREAD_PROCESS_DEFAULT, + count, + }; + return 0; + } else { + assert(!"bad count"); + return EINVAL; + } } diff --git a/libc/thread/pthread_barrier_wait.c b/libc/intrin/pthread_barrier_wait.c similarity index 74% rename from libc/thread/pthread_barrier_wait.c rename to libc/intrin/pthread_barrier_wait.c index cfd3a891f..d1931b4e9 100644 --- a/libc/thread/pthread_barrier_wait.c +++ b/libc/intrin/pthread_barrier_wait.c @@ -17,12 +17,9 @@ │ PERFORMANCE OF THIS SOFTWARE. │ ╚─────────────────────────────────────────────────────────────────────────────*/ #include "libc/intrin/atomic.h" -#include "libc/intrin/kprintf.h" +#include "libc/intrin/futex.internal.h" +#include "libc/intrin/pthread.h" #include "libc/limits.h" -#include "libc/thread/thread.h" -#include "third_party/nsync/counter.h" -#include "third_party/nsync/futex.internal.h" -#include "third_party/nsync/time.h" /** * Waits for all threads to arrive at barrier. @@ -36,10 +33,22 @@ * thread which was the last arrival, or an errno on error */ int pthread_barrier_wait(pthread_barrier_t *barrier) { - if (nsync_counter_add(barrier->_nsync, -1)) { - nsync_counter_wait(barrier->_nsync, nsync_time_no_deadline); - return 0; - } else { + if (atomic_fetch_add(&barrier->waits, 1) + 1 == barrier->count) { + if (atomic_fetch_add(&barrier->waits, 1) + 1 < barrier->count * 2) { + atomic_store_explicit(&barrier->popped, 1, memory_order_relaxed); + do { + _futex_wake(&barrier->popped, INT_MAX, barrier->pshared); + } while (atomic_load_explicit(&barrier->waits, memory_order_relaxed) < + barrier->count * 2); + atomic_store_explicit(&barrier->popped, 0, memory_order_relaxed); + } + atomic_store_explicit(&barrier->waits, 0, memory_order_relaxed); return PTHREAD_BARRIER_SERIAL_THREAD; } + do { + _futex_wait(&barrier->popped, 0, barrier->pshared, 0); + } while (atomic_load_explicit(&barrier->waits, memory_order_relaxed) < + barrier->count); + atomic_fetch_add(&barrier->waits, 1); + return 0; } diff --git a/libc/thread/pthread_barrierattr_destroy.c b/libc/intrin/pthread_barrierattr_destroy.c similarity index 98% rename from libc/thread/pthread_barrierattr_destroy.c rename to libc/intrin/pthread_barrierattr_destroy.c index 3777f2b3f..9a51e9f7f 100644 --- a/libc/thread/pthread_barrierattr_destroy.c +++ b/libc/intrin/pthread_barrierattr_destroy.c @@ -16,7 +16,7 @@ │ TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR │ │ PERFORMANCE OF THIS SOFTWARE. │ ╚─────────────────────────────────────────────────────────────────────────────*/ -#include "libc/thread/thread.h" +#include "libc/intrin/pthread.h" #include "libc/str/str.h" /** diff --git a/libc/thread/pthread_barrierattr_getpshared.c b/libc/intrin/pthread_barrierattr_getpshared.c similarity index 94% rename from libc/thread/pthread_barrierattr_getpshared.c rename to libc/intrin/pthread_barrierattr_getpshared.c index bd2c547b0..63e9228a8 100644 --- a/libc/thread/pthread_barrierattr_getpshared.c +++ b/libc/intrin/pthread_barrierattr_getpshared.c @@ -16,14 +16,14 @@ │ TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR │ │ PERFORMANCE OF THIS SOFTWARE. │ ╚─────────────────────────────────────────────────────────────────────────────*/ -#include "libc/thread/thread.h" +#include "libc/intrin/pthread.h" /** * Gets barrier process sharing. * * @param pshared is set to one of the following - * - `PTHREAD_PROCESS_PRIVATE` (default) - * - `PTHREAD_PROCESS_SHARED` (unsupported) + * - `PTHREAD_PROCESS_SHARED` + * - `PTHREAD_PROCESS_PRIVATE` * @return 0 on success, or error on failure */ int pthread_barrierattr_getpshared(const pthread_barrierattr_t *attr, diff --git a/libc/thread/pthread_barrierattr_init.c b/libc/intrin/pthread_barrierattr_init.c similarity index 96% rename from libc/thread/pthread_barrierattr_init.c rename to libc/intrin/pthread_barrierattr_init.c index b009911e2..9a79d1460 100644 --- a/libc/thread/pthread_barrierattr_init.c +++ b/libc/intrin/pthread_barrierattr_init.c @@ -16,7 +16,7 @@ │ TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR │ │ PERFORMANCE OF THIS SOFTWARE. │ ╚─────────────────────────────────────────────────────────────────────────────*/ -#include "libc/thread/thread.h" +#include "libc/intrin/pthread.h" /** * Initializes barrier attributes. @@ -24,6 +24,6 @@ * @return 0 on success, or error on failure */ int pthread_barrierattr_init(pthread_barrierattr_t *attr) { - *attr = 0; + *attr = PTHREAD_PROCESS_DEFAULT; return 0; } diff --git a/libc/thread/pthread_barrierattr_setpshared.c b/libc/intrin/pthread_barrierattr_setpshared.c similarity index 94% rename from libc/thread/pthread_barrierattr_setpshared.c rename to libc/intrin/pthread_barrierattr_setpshared.c index 4ae2adbb6..d10ee6aac 100644 --- a/libc/thread/pthread_barrierattr_setpshared.c +++ b/libc/intrin/pthread_barrierattr_setpshared.c @@ -17,19 +17,20 @@ │ PERFORMANCE OF THIS SOFTWARE. │ ╚─────────────────────────────────────────────────────────────────────────────*/ #include "libc/errno.h" -#include "libc/thread/thread.h" +#include "libc/intrin/pthread.h" /** * Sets barrier process sharing. * * @param pshared can be one of - * - `PTHREAD_PROCESS_PRIVATE` (default) - * - `PTHREAD_PROCESS_SHARED` (unsupported) + * - `PTHREAD_PROCESS_SHARED` + * - `PTHREAD_PROCESS_PRIVATE` * @return 0 on success, or error on failure * @raises EINVAL if `pshared` is invalid */ int pthread_barrierattr_setpshared(pthread_barrierattr_t *attr, int pshared) { switch (pshared) { + case PTHREAD_PROCESS_SHARED: case PTHREAD_PROCESS_PRIVATE: *attr = pshared; return 0; diff --git a/libc/intrin/pthread_cond_broadcast.c b/libc/intrin/pthread_cond_broadcast.c new file mode 100644 index 000000000..5c1cb8264 --- /dev/null +++ b/libc/intrin/pthread_cond_broadcast.c @@ -0,0 +1,96 @@ +/*-*- mode:c;indent-tabs-mode:nil;c-basic-offset:2;tab-width:8;coding:utf-8 -*-│ +│vi: set net ft=c ts=2 sts=2 sw=2 fenc=utf-8 :vi│ +╞══════════════════════════════════════════════════════════════════════════════╡ +│ Copyright 2022 Justine Alexandra Roberts Tunney │ +│ │ +│ Permission to use, copy, modify, and/or distribute this software for │ +│ any purpose with or without fee is hereby granted, provided that the │ +│ above copyright notice and this permission notice appear in all copies. │ +│ │ +│ THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL │ +│ WARRANTIES WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED │ +│ WARRANTIES OF MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE │ +│ AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL │ +│ DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR │ +│ PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER │ +│ TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR │ +│ PERFORMANCE OF THIS SOFTWARE. │ +╚─────────────────────────────────────────────────────────────────────────────*/ +#include "libc/dce.h" +#include "libc/intrin/atomic.h" +#include "libc/intrin/futex.internal.h" +#include "libc/intrin/pthread.h" +#include "libc/limits.h" + +static dontinline int pthread_cond_signal_impl(pthread_cond_t *cond, int n) { + if (atomic_load_explicit(&cond->waits, memory_order_relaxed)) { + atomic_fetch_add(&cond->seq, 1); + if (IsLinux() || IsOpenbsd()) { + _futex_wake(&cond->seq, n, cond->pshared); + } + } + return 0; +} + +/** + * Wakes at least one thread waiting on condition, e.g. + * + * pthread_cond_t cond = PTHREAD_COND_INITIALIZER; + * pthread_mutex_t lock = PTHREAD_MUTEX_INITIALIZER; + * + * // thread pool waiters + * pthread_mutex_lock(&lock); + * pthread_cond_wait(&cond, &lock); + * pthread_mutex_unlock(&lock); + * + * // waker upper + * pthread_mutex_lock(&lock); + * pthread_cond_signal(&cond); + * pthread_mutex_unlock(&lock); + * + * This function has no effect if there aren't any threads currently + * waiting on the condition. + * + * @return 0 on success, or errno on error + * @see pthread_cond_broadcast + * @see pthread_cond_wait + */ +int pthread_cond_signal(pthread_cond_t *cond) { + return pthread_cond_signal_impl(cond, 1); +} + +/** + * Wakes all threads waiting on condition, e.g. + * + * pthread_mutex_t lock; + * pthread_mutexattr_t mattr; + * pthread_mutexattr_init(&mattr); + * pthread_mutexattr_settype(&mattr, PTHREAD_MUTEX_ERRORCHECK); + * pthread_mutex_init(&lock, &mattr); + * + * pthread_cond_t cond; + * pthread_condattr_t cattr; + * pthread_condattr_init(&cattr); + * pthread_condattr_setpshared(&cattr, PTHREAD_PROCESS_SHARED); + * pthread_cond_init(&cond, &cattr); + * + * // waiting threads + * CHECK_EQ(0, pthread_mutex_lock(&lock)); + * CHECK_EQ(0, pthread_cond_wait(&cond, &lock)); + * pthread_mutex_unlock(&lock); + * + * // notifying thread + * CHECK_EQ(0, pthread_mutex_lock(&lock)); + * pthread_cond_broadcast(&cond); + * pthread_mutex_unlock(&lock); + * + * This function has no effect if there aren't any threads currently + * waiting on the condition. + * + * @return 0 on success, or errno on error + * @see pthread_cond_signal + * @see pthread_cond_wait + */ +int pthread_cond_broadcast(pthread_cond_t *cond) { + return pthread_cond_signal_impl(cond, INT_MAX); +} diff --git a/libc/thread/pthread_cond_destroy.c b/libc/intrin/pthread_cond_destroy.c similarity index 93% rename from libc/thread/pthread_cond_destroy.c rename to libc/intrin/pthread_cond_destroy.c index 579b2399a..0fc53ba88 100644 --- a/libc/thread/pthread_cond_destroy.c +++ b/libc/intrin/pthread_cond_destroy.c @@ -16,8 +16,10 @@ │ TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR │ │ PERFORMANCE OF THIS SOFTWARE. │ ╚─────────────────────────────────────────────────────────────────────────────*/ +#include "libc/assert.h" +#include "libc/errno.h" +#include "libc/intrin/pthread.h" #include "libc/str/str.h" -#include "libc/thread/thread.h" /** * Destroys condition. @@ -26,6 +28,10 @@ * @raise EINVAL if threads are still waiting on condition */ int pthread_cond_destroy(pthread_cond_t *cond) { + if (cond->waits) { + assert(!"deadlock"); + return EINVAL; + } memset(cond, -1, sizeof(*cond)); return 0; } diff --git a/libc/thread/pthread_cond_init.c b/libc/intrin/pthread_cond_init.c similarity index 95% rename from libc/thread/pthread_cond_init.c rename to libc/intrin/pthread_cond_init.c index aa5e8e8b7..f683b6350 100644 --- a/libc/thread/pthread_cond_init.c +++ b/libc/intrin/pthread_cond_init.c @@ -16,7 +16,7 @@ │ TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR │ │ PERFORMANCE OF THIS SOFTWARE. │ ╚─────────────────────────────────────────────────────────────────────────────*/ -#include "libc/thread/thread.h" +#include "libc/intrin/pthread.h" /** * Initializes condition. @@ -25,6 +25,6 @@ * @return 0 on success, or error number on failure */ int pthread_cond_init(pthread_cond_t *cond, const pthread_condattr_t *attr) { - *cond = (pthread_cond_t){0}; + *cond = (pthread_cond_t){attr ? *attr : PTHREAD_PROCESS_DEFAULT}; return 0; } diff --git a/libc/thread/pthread_condattr_destroy.c b/libc/intrin/pthread_condattr_destroy.c similarity index 98% rename from libc/thread/pthread_condattr_destroy.c rename to libc/intrin/pthread_condattr_destroy.c index abf437dd5..7ea243c59 100644 --- a/libc/thread/pthread_condattr_destroy.c +++ b/libc/intrin/pthread_condattr_destroy.c @@ -16,7 +16,7 @@ │ TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR │ │ PERFORMANCE OF THIS SOFTWARE. │ ╚─────────────────────────────────────────────────────────────────────────────*/ -#include "libc/thread/thread.h" +#include "libc/intrin/pthread.h" #include "libc/str/str.h" /** diff --git a/libc/thread/pthread_condattr_getpshared.c b/libc/intrin/pthread_condattr_getpshared.c similarity index 96% rename from libc/thread/pthread_condattr_getpshared.c rename to libc/intrin/pthread_condattr_getpshared.c index dab698903..b340781ff 100644 --- a/libc/thread/pthread_condattr_getpshared.c +++ b/libc/intrin/pthread_condattr_getpshared.c @@ -16,14 +16,14 @@ │ TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR │ │ PERFORMANCE OF THIS SOFTWARE. │ ╚─────────────────────────────────────────────────────────────────────────────*/ -#include "libc/thread/thread.h" +#include "libc/intrin/pthread.h" /** * Gets condition process sharing. * * @param pshared is set to one of the following - * - `PTHREAD_PROCESS_PRIVATE` (default) * - `PTHREAD_PROCESS_SHARED` + * - `PTHREAD_PROCESS_PRIVATE` * @return 0 on success, or error on failure */ int pthread_condattr_getpshared(const pthread_condattr_t *attr, int *pshared) { diff --git a/libc/thread/pthread_condattr_init.c b/libc/intrin/pthread_condattr_init.c similarity index 96% rename from libc/thread/pthread_condattr_init.c rename to libc/intrin/pthread_condattr_init.c index 7e61998fd..40a35f179 100644 --- a/libc/thread/pthread_condattr_init.c +++ b/libc/intrin/pthread_condattr_init.c @@ -16,7 +16,7 @@ │ TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR │ │ PERFORMANCE OF THIS SOFTWARE. │ ╚─────────────────────────────────────────────────────────────────────────────*/ -#include "libc/thread/thread.h" +#include "libc/intrin/pthread.h" /** * Initializes condition attributes. @@ -24,6 +24,6 @@ * @return 0 on success, or error on failure */ int pthread_condattr_init(pthread_condattr_t *attr) { - *attr = 0; + *attr = PTHREAD_PROCESS_DEFAULT; return 0; } diff --git a/libc/thread/pthread_condattr_setpshared.c b/libc/intrin/pthread_condattr_setpshared.c similarity index 94% rename from libc/thread/pthread_condattr_setpshared.c rename to libc/intrin/pthread_condattr_setpshared.c index ccd7ae242..9c8929a27 100644 --- a/libc/thread/pthread_condattr_setpshared.c +++ b/libc/intrin/pthread_condattr_setpshared.c @@ -17,19 +17,20 @@ │ PERFORMANCE OF THIS SOFTWARE. │ ╚─────────────────────────────────────────────────────────────────────────────*/ #include "libc/errno.h" -#include "libc/thread/thread.h" +#include "libc/intrin/pthread.h" /** * Sets condition process sharing. * * @param pshared can be one of - * - `PTHREAD_PROCESS_PRIVATE` (default) - * - `PTHREAD_PROCESS_SHARED` (unsupported) + * - `PTHREAD_PROCESS_SHARED` + * - `PTHREAD_PROCESS_PRIVATE` * @return 0 on success, or error on failure * @raises EINVAL if `pshared` is invalid */ int pthread_condattr_setpshared(pthread_condattr_t *attr, int pshared) { switch (pshared) { + case PTHREAD_PROCESS_SHARED: case PTHREAD_PROCESS_PRIVATE: *attr = pshared; return 0; diff --git a/libc/thread/pthread_mutex_destroy.c b/libc/intrin/pthread_mutex_destroy.c similarity index 92% rename from libc/thread/pthread_mutex_destroy.c rename to libc/intrin/pthread_mutex_destroy.c index 87f4efa72..5a5312910 100644 --- a/libc/thread/pthread_mutex_destroy.c +++ b/libc/intrin/pthread_mutex_destroy.c @@ -16,8 +16,10 @@ │ TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR │ │ PERFORMANCE OF THIS SOFTWARE. │ ╚─────────────────────────────────────────────────────────────────────────────*/ +#include "libc/assert.h" +#include "libc/errno.h" +#include "libc/intrin/pthread.h" #include "libc/str/str.h" -#include "libc/thread/thread.h" /** * Destroys mutex. @@ -26,6 +28,10 @@ * @raise EINVAL if mutex is locked in our implementation */ int pthread_mutex_destroy(pthread_mutex_t *mutex) { + if (mutex->lock || mutex->waits) { + assert(!"deadlock"); + return EINVAL; + } memset(mutex, -1, sizeof(*mutex)); return 0; } diff --git a/libc/intrin/pthread_mutex_init.c b/libc/intrin/pthread_mutex_init.c index f7fda57cd..0645c6a0e 100644 --- a/libc/intrin/pthread_mutex_init.c +++ b/libc/intrin/pthread_mutex_init.c @@ -16,7 +16,7 @@ │ TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR │ │ PERFORMANCE OF THIS SOFTWARE. │ ╚─────────────────────────────────────────────────────────────────────────────*/ -#include "libc/thread/thread.h" +#include "libc/intrin/pthread.h" /** * Initializes mutex. @@ -27,8 +27,8 @@ int pthread_mutex_init(pthread_mutex_t *mutex, const pthread_mutexattr_t *attr) { *mutex = (pthread_mutex_t){ - ._type = attr ? attr->_type : 0, - ._pshared = attr ? attr->_pshared : 0, + attr ? attr->type : PTHREAD_MUTEX_DEFAULT, + attr ? attr->pshared : PTHREAD_PROCESS_DEFAULT, }; return 0; } diff --git a/libc/intrin/pthread_mutex_lock.c b/libc/intrin/pthread_mutex_lock.c index 2d4d04818..374f97a2d 100644 --- a/libc/intrin/pthread_mutex_lock.c +++ b/libc/intrin/pthread_mutex_lock.c @@ -16,14 +16,32 @@ │ TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR │ │ PERFORMANCE OF THIS SOFTWARE. │ ╚─────────────────────────────────────────────────────────────────────────────*/ +#include "libc/assert.h" #include "libc/calls/calls.h" #include "libc/errno.h" +#include "libc/intrin/asmflag.h" #include "libc/intrin/atomic.h" -#include "libc/intrin/likely.h" -#include "libc/intrin/weaken.h" -#include "libc/thread/thread.h" -#include "libc/thread/tls.h" -#include "third_party/nsync/mu.h" +#include "libc/intrin/futex.internal.h" +#include "libc/intrin/pthread.h" +#include "libc/linux/futex.h" +#include "libc/nexgen32e/threaded.h" +#include "libc/sysv/consts/futex.h" +#include "libc/sysv/consts/nr.h" + +static int pthread_mutex_lock_spin(pthread_mutex_t *mutex, int expect, + int tries) { + if (tries < 7) { + volatile int i; + for (i = 0; i != 1 << tries; i++) { + } + tries++; + } else { + atomic_fetch_add(&mutex->waits, 1); + _futex_wait(&mutex->lock, expect, mutex->pshared, &(struct timespec){1}); + atomic_fetch_sub(&mutex->waits, 1); + } + return tries; +} /** * Locks mutex. @@ -56,58 +74,57 @@ * pthread_mutex_unlock(&lock); * pthread_mutex_destroy(&lock); * + * Alternatively, Cosmopolitan lets you do the folllowing instead: + * + * pthread_mutex_t lock = {PTHREAD_MUTEX_RECURSIVE}; + * pthread_mutex_lock(&lock); + * // do work... + * pthread_mutex_unlock(&lock); + * * @return 0 on success, or error number on failure * @see pthread_spin_lock() */ int pthread_mutex_lock(pthread_mutex_t *mutex) { - int c, d, t; - - if (LIKELY(__tls_enabled && // - mutex->_type == PTHREAD_MUTEX_NORMAL && // - mutex->_pshared == PTHREAD_PROCESS_PRIVATE && // - weaken(nsync_mu_lock))) { - weaken(nsync_mu_lock)((nsync_mu *)mutex); - return 0; - } - - t = gettid(); - if (mutex->_type == PTHREAD_MUTEX_ERRORCHECK) { - c = atomic_load_explicit(&mutex->_lock, memory_order_relaxed); - if ((c & 0x000fffff) == t) { - return EDEADLK; - } - } - - for (;;) { - c = 0; - d = 0x10100000 | t; - if (atomic_compare_exchange_weak_explicit( - &mutex->_lock, &c, d, memory_order_acquire, memory_order_relaxed)) { - break; - } else { - if ((c & 0x000fffff) == t) { - if ((c & 0x0ff00000) < 0x0ff00000) { - c = atomic_fetch_add_explicit(&mutex->_lock, 0x00100000, - memory_order_relaxed); + int c, me, owner, tries; + switch (mutex->type) { + case PTHREAD_MUTEX_NORMAL: + // From Futexes Are Tricky Version 1.1 § Mutex, Take 3; + // Ulrich Drepper, Red Hat Incorporated, June 27, 2004. + c = 0; + if (!atomic_compare_exchange_strong_explicit(&mutex->lock, &c, 1, + memory_order_acquire, + memory_order_relaxed)) { + if (c != 2) { + c = atomic_exchange_explicit(&mutex->lock, 2, memory_order_acquire); + } + while (c) { + _futex_wait(&mutex->lock, 2, mutex->pshared, 0); + c = atomic_exchange_explicit(&mutex->lock, 2, memory_order_acquire); + } + } + return 0; + case PTHREAD_MUTEX_RECURSIVE: + case PTHREAD_MUTEX_ERRORCHECK: + for (tries = 0, me = gettid();;) { + owner = atomic_load_explicit(&mutex->lock, memory_order_relaxed); + if (!owner && atomic_compare_exchange_weak_explicit( + &mutex->lock, &owner, me, memory_order_acquire, + memory_order_relaxed)) { break; - } else { - return EAGAIN; + } else if (owner == me) { + if (mutex->type != PTHREAD_MUTEX_ERRORCHECK) { + break; + } else { + assert(!"deadlock"); + return EDEADLK; + } } + tries = pthread_mutex_lock_spin(mutex, owner, tries); } - if ((c & 0xf0000000) == 0x10000000) { - d = 0x20000000 | c; - if (atomic_compare_exchange_weak_explicit(&mutex->_lock, &c, d, - memory_order_acquire, - memory_order_relaxed)) { - c = d; - } - } - if ((c & 0xf0000000) == 0x20000000) { - // _futex_wait(&mutex->_lock, c, mutex->_pshared, 0); - pthread_yield(); - } - } + ++mutex->reent; + return 0; + default: + assert(!"badlock"); + return EINVAL; } - - return 0; } diff --git a/libc/intrin/pthread_mutex_trylock.c b/libc/intrin/pthread_mutex_trylock.c index 9621115e0..6d1a20151 100644 --- a/libc/intrin/pthread_mutex_trylock.c +++ b/libc/intrin/pthread_mutex_trylock.c @@ -20,11 +20,7 @@ #include "libc/calls/calls.h" #include "libc/errno.h" #include "libc/intrin/atomic.h" -#include "libc/intrin/likely.h" -#include "libc/intrin/weaken.h" -#include "libc/thread/thread.h" -#include "libc/thread/tls.h" -#include "third_party/nsync/mu.h" +#include "libc/intrin/pthread.h" /** * Locks mutex if it isn't locked already. @@ -37,42 +33,36 @@ * @raise ENOTRECOVERABLE if `mutex` is corrupted */ int pthread_mutex_trylock(pthread_mutex_t *mutex) { - int c, d, t; - - if (LIKELY(__tls_enabled && // - mutex->_type == PTHREAD_MUTEX_NORMAL && // - mutex->_pshared == PTHREAD_PROCESS_PRIVATE && // - weaken(nsync_mu_trylock))) { - if (weaken(nsync_mu_trylock)((nsync_mu *)mutex)) { + int c, me, owner; + switch (mutex->type) { + case PTHREAD_MUTEX_NORMAL: + c = 0; + if (atomic_compare_exchange_strong_explicit(&mutex->lock, &c, 1, + memory_order_acquire, + memory_order_relaxed)) { + return 0; + } else { + return EBUSY; + } + case PTHREAD_MUTEX_RECURSIVE: + case PTHREAD_MUTEX_ERRORCHECK: + owner = 0; + me = gettid(); + if (!atomic_compare_exchange_strong_explicit(&mutex->lock, &owner, me, + memory_order_acquire, + memory_order_relaxed)) { + if (owner == me) { + if (mutex->type == PTHREAD_MUTEX_ERRORCHECK) { + return EBUSY; + } + } else { + return EBUSY; + } + } + ++mutex->reent; return 0; - } else { - return EBUSY; - } + default: + assert(!"badlock"); + return ENOTRECOVERABLE; } - - if (mutex->_type == PTHREAD_MUTEX_NORMAL) { - c = 0; - if (atomic_compare_exchange_strong_explicit( - &mutex->_lock, &c, 1, memory_order_acquire, memory_order_relaxed)) { - return 0; - } else { - return EBUSY; - } - } - - c = 0; - t = gettid(); - d = 0x10100000 | t; - if (!atomic_compare_exchange_strong_explicit( - &mutex->_lock, &c, d, memory_order_acquire, memory_order_relaxed)) { - if ((c & 0x000fffff) != t || mutex->_type == PTHREAD_MUTEX_ERRORCHECK) { - return EBUSY; - } - if ((c & 0x0ff00000) == 0x0ff00000) { - return EAGAIN; - } - atomic_fetch_add_explicit(&mutex->_lock, 0x00100000, memory_order_relaxed); - } - - return 0; } diff --git a/libc/intrin/pthread_mutex_unlock.c b/libc/intrin/pthread_mutex_unlock.c index d0e65b732..430b9c8c6 100644 --- a/libc/intrin/pthread_mutex_unlock.c +++ b/libc/intrin/pthread_mutex_unlock.c @@ -16,14 +16,13 @@ │ TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR │ │ PERFORMANCE OF THIS SOFTWARE. │ ╚─────────────────────────────────────────────────────────────────────────────*/ +#include "libc/assert.h" #include "libc/calls/calls.h" +#include "libc/dce.h" #include "libc/errno.h" #include "libc/intrin/atomic.h" -#include "libc/intrin/likely.h" -#include "libc/intrin/weaken.h" -#include "libc/thread/thread.h" -#include "libc/thread/tls.h" -#include "third_party/nsync/mu.h" +#include "libc/intrin/futex.internal.h" +#include "libc/intrin/pthread.h" /** * Releases mutex. @@ -32,32 +31,35 @@ * @raises EPERM if in error check mode and not owned by caller */ int pthread_mutex_unlock(pthread_mutex_t *mutex) { - int c, t; - - if (LIKELY(__tls_enabled && // - mutex->_type == PTHREAD_MUTEX_NORMAL && // - mutex->_pshared == PTHREAD_PROCESS_PRIVATE && // - weaken(nsync_mu_unlock))) { - weaken(nsync_mu_unlock)((nsync_mu *)mutex); - return 0; + int c, me, owner; + switch (mutex->type) { + case PTHREAD_MUTEX_NORMAL: + // From Futexes Are Tricky Version 1.1 § Mutex, Take 3; + // Ulrich Drepper, Red Hat Incorporated, June 27, 2004. + if ((c = atomic_fetch_sub_explicit(&mutex->lock, 1, + memory_order_release)) != 1) { + atomic_store_explicit(&mutex->lock, 0, memory_order_release); + _futex_wake(&mutex->lock, 1, mutex->pshared); + } + return 0; + case PTHREAD_MUTEX_ERRORCHECK: + me = gettid(); + owner = atomic_load_explicit(&mutex->lock, memory_order_relaxed); + if (owner != me) { + assert(!"permlock"); + return EPERM; + } + // fallthrough + case PTHREAD_MUTEX_RECURSIVE: + if (--mutex->reent) return 0; + atomic_store_explicit(&mutex->lock, 0, memory_order_relaxed); + if ((IsLinux() || IsOpenbsd()) && + atomic_load_explicit(&mutex->waits, memory_order_relaxed) > 0) { + return _futex_wake(&mutex->lock, 1, mutex->pshared); + } + return 0; + default: + assert(!"badlock"); + return EINVAL; } - - t = gettid(); - if (mutex->_type == PTHREAD_MUTEX_ERRORCHECK) { - c = atomic_load_explicit(&mutex->_lock, memory_order_relaxed); - if ((c & 0x000fffff) != t) { - return EPERM; - } - } - - c = atomic_fetch_sub(&mutex->_lock, 0x00100000); - if ((c & 0x0ff00000) == 0x00100000) { - atomic_store_explicit(&mutex->_lock, 0, memory_order_release); - if ((c & 0xf0000000) == 0x20000000) { - // _futex_wake(&mutex->_lock, 1, mutex->_pshared); - pthread_yield(); - } - } - - return 0; } diff --git a/libc/thread/pthread_mutexattr_destroy.c b/libc/intrin/pthread_mutexattr_destroy.c similarity index 98% rename from libc/thread/pthread_mutexattr_destroy.c rename to libc/intrin/pthread_mutexattr_destroy.c index 49df496c1..a6fc17e3e 100644 --- a/libc/thread/pthread_mutexattr_destroy.c +++ b/libc/intrin/pthread_mutexattr_destroy.c @@ -16,7 +16,7 @@ │ TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR │ │ PERFORMANCE OF THIS SOFTWARE. │ ╚─────────────────────────────────────────────────────────────────────────────*/ -#include "libc/thread/thread.h" +#include "libc/intrin/pthread.h" #include "libc/str/str.h" /** diff --git a/libc/thread/pthread_mutexattr_getpshared.c b/libc/intrin/pthread_mutexattr_getpshared.c similarity index 95% rename from libc/thread/pthread_mutexattr_getpshared.c rename to libc/intrin/pthread_mutexattr_getpshared.c index c521b91ce..8d76df79b 100644 --- a/libc/thread/pthread_mutexattr_getpshared.c +++ b/libc/intrin/pthread_mutexattr_getpshared.c @@ -16,18 +16,18 @@ │ TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR │ │ PERFORMANCE OF THIS SOFTWARE. │ ╚─────────────────────────────────────────────────────────────────────────────*/ -#include "libc/thread/thread.h" +#include "libc/intrin/pthread.h" /** * Gets mutex process sharing. * * @param pshared is set to one of the following - * - `PTHREAD_PROCESS_PRIVATE` (default) * - `PTHREAD_PROCESS_SHARED` + * - `PTHREAD_PROCESS_PRIVATE` * @return 0 on success, or error on failure */ int pthread_mutexattr_getpshared(const pthread_mutexattr_t *attr, int *pshared) { - *pshared = attr->_pshared; + *pshared = attr->pshared; return 0; } diff --git a/libc/thread/pthread_mutexattr_gettype.c b/libc/intrin/pthread_mutexattr_gettype.c similarity index 97% rename from libc/thread/pthread_mutexattr_gettype.c rename to libc/intrin/pthread_mutexattr_gettype.c index 6138568e9..2c7ef3e0a 100644 --- a/libc/thread/pthread_mutexattr_gettype.c +++ b/libc/intrin/pthread_mutexattr_gettype.c @@ -16,7 +16,7 @@ │ TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR │ │ PERFORMANCE OF THIS SOFTWARE. │ ╚─────────────────────────────────────────────────────────────────────────────*/ -#include "libc/thread/thread.h" +#include "libc/intrin/pthread.h" /** * Gets mutex type. @@ -29,6 +29,6 @@ * @return 0 on success, or error on failure */ int pthread_mutexattr_gettype(const pthread_mutexattr_t *attr, int *type) { - *type = attr->_type; + *type = attr->type; return 0; } diff --git a/libc/thread/pthread_mutexattr_init.c b/libc/intrin/pthread_mutexattr_init.c similarity index 93% rename from libc/thread/pthread_mutexattr_init.c rename to libc/intrin/pthread_mutexattr_init.c index a12bec7d8..5bb326829 100644 --- a/libc/thread/pthread_mutexattr_init.c +++ b/libc/intrin/pthread_mutexattr_init.c @@ -16,13 +16,16 @@ │ TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR │ │ PERFORMANCE OF THIS SOFTWARE. │ ╚─────────────────────────────────────────────────────────────────────────────*/ -#include "libc/thread/thread.h" +#include "libc/intrin/pthread.h" /** * Initializes mutex attr. * @return 0 on success, or error number on failure */ int pthread_mutexattr_init(pthread_mutexattr_t *attr) { - *attr = (pthread_mutexattr_t){0}; + *attr = (pthread_mutexattr_t){ + PTHREAD_MUTEX_DEFAULT, + PTHREAD_PROCESS_DEFAULT, + }; return 0; } diff --git a/libc/thread/pthread_mutexattr_setpshared.c b/libc/intrin/pthread_mutexattr_setpshared.c similarity index 95% rename from libc/thread/pthread_mutexattr_setpshared.c rename to libc/intrin/pthread_mutexattr_setpshared.c index c575cbd95..8152edce2 100644 --- a/libc/thread/pthread_mutexattr_setpshared.c +++ b/libc/intrin/pthread_mutexattr_setpshared.c @@ -17,14 +17,14 @@ │ PERFORMANCE OF THIS SOFTWARE. │ ╚─────────────────────────────────────────────────────────────────────────────*/ #include "libc/errno.h" -#include "libc/thread/thread.h" +#include "libc/intrin/pthread.h" /** * Sets mutex process sharing. * * @param pshared can be one of - * - `PTHREAD_PROCESS_PRIVATE` (default) * - `PTHREAD_PROCESS_SHARED` + * - `PTHREAD_PROCESS_PRIVATE` * @return 0 on success, or error on failure * @raises EINVAL if `pshared` is invalid */ @@ -32,7 +32,7 @@ int pthread_mutexattr_setpshared(pthread_mutexattr_t *attr, int pshared) { switch (pshared) { case PTHREAD_PROCESS_SHARED: case PTHREAD_PROCESS_PRIVATE: - attr->_pshared = pshared; + attr->pshared = pshared; return 0; default: return EINVAL; diff --git a/libc/thread/pthread_mutexattr_settype.c b/libc/intrin/pthread_mutexattr_settype.c similarity index 97% rename from libc/thread/pthread_mutexattr_settype.c rename to libc/intrin/pthread_mutexattr_settype.c index 766a4b7a4..546459156 100644 --- a/libc/thread/pthread_mutexattr_settype.c +++ b/libc/intrin/pthread_mutexattr_settype.c @@ -17,7 +17,7 @@ │ PERFORMANCE OF THIS SOFTWARE. │ ╚─────────────────────────────────────────────────────────────────────────────*/ #include "libc/errno.h" -#include "libc/thread/thread.h" +#include "libc/intrin/pthread.h" /** * Sets mutex type. @@ -35,7 +35,7 @@ int pthread_mutexattr_settype(pthread_mutexattr_t *attr, int type) { case PTHREAD_MUTEX_NORMAL: case PTHREAD_MUTEX_RECURSIVE: case PTHREAD_MUTEX_ERRORCHECK: - attr->_type = type; + attr->type = type; return 0; default: return EINVAL; diff --git a/libc/intrin/pthread_once.c b/libc/intrin/pthread_once.c index dc8ce6fb7..1d13fd863 100644 --- a/libc/intrin/pthread_once.c +++ b/libc/intrin/pthread_once.c @@ -16,12 +16,11 @@ │ TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR │ │ PERFORMANCE OF THIS SOFTWARE. │ ╚─────────────────────────────────────────────────────────────────────────────*/ +#include "libc/assert.h" #include "libc/calls/calls.h" #include "libc/errno.h" #include "libc/intrin/atomic.h" -#include "libc/intrin/weaken.h" -#include "libc/thread/thread.h" -#include "third_party/nsync/once.h" +#include "libc/intrin/pthread.h" #define INIT 0 #define CALLING 1 @@ -45,30 +44,28 @@ * @return 0 on success, or errno on error */ int pthread_once(pthread_once_t *once, void init(void)) { - uint32_t old; - if (weaken(nsync_run_once)) { - weaken(nsync_run_once)((nsync_once *)once, init); - return 0; - } - switch ((old = atomic_load_explicit(&once->_lock, memory_order_relaxed))) { + char old; + switch ((old = atomic_load_explicit(&once->lock, memory_order_relaxed))) { case INIT: - if (atomic_compare_exchange_strong_explicit(&once->_lock, &old, CALLING, + if (atomic_compare_exchange_strong_explicit(&once->lock, &old, CALLING, memory_order_acquire, memory_order_relaxed)) { init(); - atomic_store(&once->_lock, FINISHED); - return 0; + atomic_store(&once->lock, FINISHED); + break; } // fallthrough case CALLING: do { pthread_yield(); - } while (atomic_load_explicit(&once->_lock, memory_order_relaxed) == + } while (atomic_load_explicit(&once->lock, memory_order_relaxed) == CALLING); - return 0; + break; case FINISHED: - return 0; + break; default: + assert(!"bad once"); return EINVAL; } + return 0; } diff --git a/libc/thread/pthread_rwlock_destroy.c b/libc/intrin/pthread_rwlock_destroy.c similarity index 93% rename from libc/thread/pthread_rwlock_destroy.c rename to libc/intrin/pthread_rwlock_destroy.c index 47bc63c10..eb0b3c906 100644 --- a/libc/thread/pthread_rwlock_destroy.c +++ b/libc/intrin/pthread_rwlock_destroy.c @@ -16,8 +16,10 @@ │ TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR │ │ PERFORMANCE OF THIS SOFTWARE. │ ╚─────────────────────────────────────────────────────────────────────────────*/ +#include "libc/assert.h" +#include "libc/errno.h" +#include "libc/intrin/pthread.h" #include "libc/str/str.h" -#include "libc/thread/thread.h" /** * Destroys read-write lock. @@ -26,6 +28,10 @@ * @raise EINVAL if any threads still hold the lock */ int pthread_rwlock_destroy(pthread_rwlock_t *rwlock) { + if (rwlock->lock) { + assert(!"deadlock"); + return EINVAL; + } memset(rwlock, -1, sizeof(*rwlock)); return 0; } diff --git a/libc/thread/pthread_rwlock_init.c b/libc/intrin/pthread_rwlock_init.c similarity index 95% rename from libc/thread/pthread_rwlock_init.c rename to libc/intrin/pthread_rwlock_init.c index c9790a9b7..392dde1c0 100644 --- a/libc/thread/pthread_rwlock_init.c +++ b/libc/intrin/pthread_rwlock_init.c @@ -16,7 +16,7 @@ │ TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR │ │ PERFORMANCE OF THIS SOFTWARE. │ ╚─────────────────────────────────────────────────────────────────────────────*/ -#include "libc/thread/thread.h" +#include "libc/intrin/pthread.h" /** * Initializes read-write lock. @@ -26,6 +26,6 @@ */ int pthread_rwlock_init(pthread_rwlock_t *rwlock, const pthread_rwlockattr_t *attr) { - *rwlock = (pthread_rwlock_t){0}; + *rwlock = (pthread_rwlock_t){attr ? *attr : PTHREAD_PROCESS_DEFAULT}; return 0; } diff --git a/libc/thread/pthread_rwlock_rdlock.c b/libc/intrin/pthread_rwlock_rdlock.c similarity index 66% rename from libc/thread/pthread_rwlock_rdlock.c rename to libc/intrin/pthread_rwlock_rdlock.c index 187b60453..e0502ab7e 100644 --- a/libc/thread/pthread_rwlock_rdlock.c +++ b/libc/intrin/pthread_rwlock_rdlock.c @@ -16,8 +16,24 @@ │ TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR │ │ PERFORMANCE OF THIS SOFTWARE. │ ╚─────────────────────────────────────────────────────────────────────────────*/ -#include "libc/thread/thread.h" -#include "third_party/nsync/mu.h" +#include "libc/intrin/atomic.h" +#include "libc/intrin/futex.internal.h" +#include "libc/intrin/pthread.h" + +static int pthread_rwlock_rdlock_spin(pthread_rwlock_t *rwlock, int expect, + int tries) { + if (tries < 7) { + volatile int i; + for (i = 0; i != 1 << tries; i++) { + } + tries++; + } else { + atomic_fetch_add(&rwlock->waits, 1); + _futex_wait(&rwlock->lock, expect, rwlock->pshared, &(struct timespec){1}); + atomic_fetch_sub(&rwlock->waits, 1); + } + return tries; +} /** * Acquires read lock on read-write lock. @@ -25,6 +41,18 @@ * @return 0 on success, or errno on error */ int pthread_rwlock_rdlock(pthread_rwlock_t *rwlock) { - nsync_mu_rlock((nsync_mu *)rwlock); - return 0; + int old, tries; + for (tries = 0;;) { + old = atomic_load_explicit(&rwlock->lock, memory_order_relaxed); + if (old >= 0) { + do { + if (atomic_compare_exchange_weak_explicit(&rwlock->lock, &old, old + 1, + memory_order_acquire, + memory_order_relaxed)) { + return 0; + } + } while (old >= 0); + } + tries = pthread_rwlock_rdlock_spin(rwlock, old, tries); + } } diff --git a/libc/thread/pthread_cond_broadcast.c b/libc/intrin/pthread_rwlock_unlock.c similarity index 62% rename from libc/thread/pthread_cond_broadcast.c rename to libc/intrin/pthread_rwlock_unlock.c index 8518a4b45..c90f9ecbd 100644 --- a/libc/thread/pthread_cond_broadcast.c +++ b/libc/intrin/pthread_rwlock_unlock.c @@ -16,27 +16,40 @@ │ TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR │ │ PERFORMANCE OF THIS SOFTWARE. │ ╚─────────────────────────────────────────────────────────────────────────────*/ -#include "libc/thread/thread.h" -#include "third_party/nsync/cv.h" +#include "libc/assert.h" +#include "libc/dce.h" +#include "libc/errno.h" +#include "libc/intrin/atomic.h" +#include "libc/intrin/futex.internal.h" +#include "libc/intrin/pthread.h" /** - * Wakes all threads waiting on condition, e.g. - * - * pthread_cond_t cond = PTHREAD_COND_INITIALIZER; - * pthread_mutex_t lock = PTHREAD_MUTEX_INITIALIZER; - * // ... - * pthread_mutex_lock(&lock); - * pthread_cond_broadcast(&cond, &lock); - * pthread_mutex_unlock(&lock); - * - * This function has no effect if there aren't any threads currently - * waiting on the condition. + * Unlocks read-write lock. * * @return 0 on success, or errno on error - * @see pthread_cond_signal - * @see pthread_cond_wait + * @raise EINVAL if lock is in a bad state */ -int pthread_cond_broadcast(pthread_cond_t *cond) { - nsync_cv_broadcast((nsync_cv *)cond); - return 0; +int pthread_rwlock_unlock(pthread_rwlock_t *rwlock) { + int old, waits; + for (;;) { + old = atomic_load_explicit(&rwlock->lock, memory_order_relaxed); + if (!old || old < -1) { + assert(!"badlock"); + return EINVAL; + } else if (old == -1 || old == 1) { + waits = atomic_load_explicit(&rwlock->waits, memory_order_relaxed); + if (atomic_compare_exchange_weak_explicit(&rwlock->lock, &old, 0, + memory_order_acquire, + memory_order_relaxed)) { + if (waits && (IsLinux() || IsOpenbsd())) { + _futex_wake(&rwlock->lock, 1, rwlock->pshared); + } + return 0; + } + } else if (atomic_compare_exchange_weak_explicit( + &rwlock->lock, &old, old - 1, memory_order_acquire, + memory_order_relaxed)) { + return 0; + } + } } diff --git a/libc/thread/pthread_rwlock_wrlock.c b/libc/intrin/pthread_rwlock_wrlock.c similarity index 66% rename from libc/thread/pthread_rwlock_wrlock.c rename to libc/intrin/pthread_rwlock_wrlock.c index f97c3ae95..f50ebc966 100644 --- a/libc/thread/pthread_rwlock_wrlock.c +++ b/libc/intrin/pthread_rwlock_wrlock.c @@ -16,8 +16,24 @@ │ TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR │ │ PERFORMANCE OF THIS SOFTWARE. │ ╚─────────────────────────────────────────────────────────────────────────────*/ -#include "libc/thread/thread.h" -#include "third_party/nsync/mu.h" +#include "libc/intrin/atomic.h" +#include "libc/intrin/futex.internal.h" +#include "libc/intrin/pthread.h" + +static int pthread_rwlock_wrlock_spin(pthread_rwlock_t *rwlock, int expect, + int tries) { + if (tries < 7) { + volatile int i; + for (i = 0; i != 1 << tries; i++) { + } + tries++; + } else { + atomic_fetch_add(&rwlock->waits, 1); + _futex_wait(&rwlock->lock, expect, rwlock->pshared, &(struct timespec){1}); + atomic_fetch_sub(&rwlock->waits, 1); + } + return tries; +} /** * Acquires write lock on read-write lock. @@ -25,7 +41,17 @@ * @return 0 on success, or errno on error */ int pthread_rwlock_wrlock(pthread_rwlock_t *rwlock) { - nsync_mu_lock((nsync_mu *)rwlock); - rwlock->_iswrite = 1; - return 0; + int old, tries; + for (tries = 0;;) { + if (!(old = atomic_load_explicit(&rwlock->lock, memory_order_relaxed))) { + do { + if (atomic_compare_exchange_weak_explicit(&rwlock->lock, &old, -1, + memory_order_acquire, + memory_order_relaxed)) { + return 0; + } + } while (!old); + } + tries = pthread_rwlock_wrlock_spin(rwlock, old, tries); + } } diff --git a/libc/thread/pthread_rwlockattr_destroy.c b/libc/intrin/pthread_rwlockattr_destroy.c similarity index 98% rename from libc/thread/pthread_rwlockattr_destroy.c rename to libc/intrin/pthread_rwlockattr_destroy.c index 624b5fb5d..b190001ce 100644 --- a/libc/thread/pthread_rwlockattr_destroy.c +++ b/libc/intrin/pthread_rwlockattr_destroy.c @@ -16,7 +16,7 @@ │ TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR │ │ PERFORMANCE OF THIS SOFTWARE. │ ╚─────────────────────────────────────────────────────────────────────────────*/ -#include "libc/thread/thread.h" +#include "libc/intrin/pthread.h" #include "libc/str/str.h" /** diff --git a/libc/thread/pthread_rwlockattr_getpshared.c b/libc/intrin/pthread_rwlockattr_getpshared.c similarity index 94% rename from libc/thread/pthread_rwlockattr_getpshared.c rename to libc/intrin/pthread_rwlockattr_getpshared.c index efafcb927..aafde2afe 100644 --- a/libc/thread/pthread_rwlockattr_getpshared.c +++ b/libc/intrin/pthread_rwlockattr_getpshared.c @@ -16,14 +16,14 @@ │ TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR │ │ PERFORMANCE OF THIS SOFTWARE. │ ╚─────────────────────────────────────────────────────────────────────────────*/ -#include "libc/thread/thread.h" +#include "libc/intrin/pthread.h" /** * Gets read-write lock process sharing. * * @param pshared is set to one of the following - * - `PTHREAD_PROCESS_PRIVATE` (default) - * - `PTHREAD_PROCESS_SHARED` (unsupported) + * - `PTHREAD_PROCESS_SHARED` + * - `PTHREAD_PROCESS_PRIVATE` * @return 0 on success, or error on failure */ int pthread_rwlockattr_getpshared(const pthread_rwlockattr_t *attr, diff --git a/libc/thread/pthread_rwlockattr_init.c b/libc/intrin/pthread_rwlockattr_init.c similarity index 96% rename from libc/thread/pthread_rwlockattr_init.c rename to libc/intrin/pthread_rwlockattr_init.c index dee6fb305..f64ded572 100644 --- a/libc/thread/pthread_rwlockattr_init.c +++ b/libc/intrin/pthread_rwlockattr_init.c @@ -16,7 +16,7 @@ │ TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR │ │ PERFORMANCE OF THIS SOFTWARE. │ ╚─────────────────────────────────────────────────────────────────────────────*/ -#include "libc/thread/thread.h" +#include "libc/intrin/pthread.h" /** * Initializes read-write lock attributes. @@ -24,6 +24,6 @@ * @return 0 on success, or error on failure */ int pthread_rwlockattr_init(pthread_rwlockattr_t *attr) { - *attr = 0; + *attr = PTHREAD_PROCESS_DEFAULT; return 0; } diff --git a/libc/thread/pthread_rwlockattr_setpshared.c b/libc/intrin/pthread_rwlockattr_setpshared.c similarity index 94% rename from libc/thread/pthread_rwlockattr_setpshared.c rename to libc/intrin/pthread_rwlockattr_setpshared.c index 5904c4d6c..2139bfd72 100644 --- a/libc/thread/pthread_rwlockattr_setpshared.c +++ b/libc/intrin/pthread_rwlockattr_setpshared.c @@ -17,19 +17,20 @@ │ PERFORMANCE OF THIS SOFTWARE. │ ╚─────────────────────────────────────────────────────────────────────────────*/ #include "libc/errno.h" -#include "libc/thread/thread.h" +#include "libc/intrin/pthread.h" /** * Sets read-write lock process sharing. * * @param pshared can be one of - * - `PTHREAD_PROCESS_PRIVATE` (default) - * - `PTHREAD_PROCESS_SHARED` (unsupported) + * - `PTHREAD_PROCESS_SHARED` + * - `PTHREAD_PROCESS_PRIVATE` * @return 0 on success, or error on failure * @raises EINVAL if `pshared` is invalid */ int pthread_rwlockattr_setpshared(pthread_rwlockattr_t *attr, int pshared) { switch (pshared) { + case PTHREAD_PROCESS_SHARED: case PTHREAD_PROCESS_PRIVATE: *attr = pshared; return 0; diff --git a/libc/thread/pthread_spin_destroy.c b/libc/intrin/pthread_spin_destroy.c similarity index 98% rename from libc/thread/pthread_spin_destroy.c rename to libc/intrin/pthread_spin_destroy.c index 18fa1bfeb..309071433 100644 --- a/libc/thread/pthread_spin_destroy.c +++ b/libc/intrin/pthread_spin_destroy.c @@ -16,7 +16,7 @@ │ TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR │ │ PERFORMANCE OF THIS SOFTWARE. │ ╚─────────────────────────────────────────────────────────────────────────────*/ -#include "libc/thread/thread.h" +#include "libc/intrin/pthread.h" /** * Destroys spin lock. diff --git a/libc/thread/pthread_spin_init.c b/libc/intrin/pthread_spin_init.c similarity index 98% rename from libc/thread/pthread_spin_init.c rename to libc/intrin/pthread_spin_init.c index d8ca27cc1..753d4e956 100644 --- a/libc/thread/pthread_spin_init.c +++ b/libc/intrin/pthread_spin_init.c @@ -16,7 +16,7 @@ │ TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR │ │ PERFORMANCE OF THIS SOFTWARE. │ ╚─────────────────────────────────────────────────────────────────────────────*/ -#include "libc/thread/thread.h" +#include "libc/intrin/pthread.h" /** * Initializes spin lock. diff --git a/libc/thread/pthread_spin_lock.c b/libc/intrin/pthread_spin_lock.c similarity index 98% rename from libc/thread/pthread_spin_lock.c rename to libc/intrin/pthread_spin_lock.c index 36ced2007..dca24fb7b 100644 --- a/libc/thread/pthread_spin_lock.c +++ b/libc/intrin/pthread_spin_lock.c @@ -16,7 +16,7 @@ │ TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR │ │ PERFORMANCE OF THIS SOFTWARE. │ ╚─────────────────────────────────────────────────────────────────────────────*/ -#include "libc/thread/thread.h" +#include "libc/intrin/pthread.h" /** * Acquires spin lock. diff --git a/libc/thread/pthread_spin_trylock.c b/libc/intrin/pthread_spin_trylock.c similarity index 98% rename from libc/thread/pthread_spin_trylock.c rename to libc/intrin/pthread_spin_trylock.c index 36bcb55c9..7936b2e8b 100644 --- a/libc/thread/pthread_spin_trylock.c +++ b/libc/intrin/pthread_spin_trylock.c @@ -16,7 +16,7 @@ │ TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR │ │ PERFORMANCE OF THIS SOFTWARE. │ ╚─────────────────────────────────────────────────────────────────────────────*/ -#include "libc/thread/thread.h" +#include "libc/intrin/pthread.h" /** * Acquires spin lock if available. diff --git a/libc/thread/pthread_spin_unlock.c b/libc/intrin/pthread_spin_unlock.c similarity index 98% rename from libc/thread/pthread_spin_unlock.c rename to libc/intrin/pthread_spin_unlock.c index 54fafbd18..ba3318c6f 100644 --- a/libc/thread/pthread_spin_unlock.c +++ b/libc/intrin/pthread_spin_unlock.c @@ -16,7 +16,7 @@ │ TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR │ │ PERFORMANCE OF THIS SOFTWARE. │ ╚─────────────────────────────────────────────────────────────────────────────*/ -#include "libc/thread/thread.h" +#include "libc/intrin/pthread.h" /** * Releases spin lock. diff --git a/libc/intrin/pthread_yield.c b/libc/intrin/pthread_yield.c index 273eabde4..322e0799c 100644 --- a/libc/intrin/pthread_yield.c +++ b/libc/intrin/pthread_yield.c @@ -18,7 +18,7 @@ ╚─────────────────────────────────────────────────────────────────────────────*/ #include "libc/calls/calls.h" #include "libc/calls/strace.internal.h" -#include "libc/thread/thread.h" +#include "libc/intrin/pthread.h" /** * Yields current thread's remaining timeslice to operating system. diff --git a/libc/intrin/quick_exit.c b/libc/intrin/quick_exit.c index fff957397..40532c777 100644 --- a/libc/intrin/quick_exit.c +++ b/libc/intrin/quick_exit.c @@ -16,8 +16,12 @@ │ TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR │ │ PERFORMANCE OF THIS SOFTWARE. │ ╚─────────────────────────────────────────────────────────────────────────────*/ -#include "libc/calls/strace.internal.h" #include "libc/intrin/weaken.h" +#include "libc/calls/strace.internal.h" +#include "libc/dce.h" +#include "libc/nt/console.h" +#include "libc/nt/process.h" +#include "libc/nt/runtime.h" #include "libc/runtime/internal.h" #include "libc/runtime/runtime.h" #include "libc/stdio/stdio.h" diff --git a/libc/intrin/rand64.c b/libc/intrin/rand64.c index 3d48c6b94..6b915c2a1 100644 --- a/libc/intrin/rand64.c +++ b/libc/intrin/rand64.c @@ -17,9 +17,9 @@ │ PERFORMANCE OF THIS SOFTWARE. │ ╚─────────────────────────────────────────────────────────────────────────────*/ #include "libc/calls/_getauxval.internal.h" -#include "libc/thread/thread.h" +#include "libc/intrin/pthread.h" #include "libc/nexgen32e/rdtsc.h" -#include "libc/thread/tls.h" +#include "libc/nexgen32e/threaded.h" #include "libc/runtime/internal.h" #include "libc/runtime/runtime.h" #include "libc/str/str.h" diff --git a/libc/intrin/tlsisrequired.c b/libc/intrin/tlsisrequired.c index 08fb4d41a..a428d3144 100644 --- a/libc/intrin/tlsisrequired.c +++ b/libc/intrin/tlsisrequired.c @@ -16,9 +16,9 @@ │ TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR │ │ PERFORMANCE OF THIS SOFTWARE. │ ╚─────────────────────────────────────────────────────────────────────────────*/ -#include "libc/thread/tls.h" +#include "libc/nexgen32e/threaded.h" -void __require_tls(void) { +void TlsIsRequired(void) { if (!__tls_enabled) { notpossible; } diff --git a/libc/thread/wait0.c b/libc/intrin/wait0.c similarity index 75% rename from libc/thread/wait0.c rename to libc/intrin/wait0.c index 380eb5718..0bfa2bf12 100644 --- a/libc/thread/wait0.c +++ b/libc/intrin/wait0.c @@ -16,17 +16,13 @@ │ TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR │ │ PERFORMANCE OF THIS SOFTWARE. │ ╚─────────────────────────────────────────────────────────────────────────────*/ -#include "libc/calls/strace.internal.h" -#include "libc/calls/struct/timespec.h" +#include "libc/calls/calls.h" #include "libc/dce.h" -#include "libc/errno.h" #include "libc/intrin/atomic.h" -#include "libc/intrin/describeflags.internal.h" -#include "libc/sysv/consts/futex.h" -#include "libc/thread/thread.h" -#include "libc/thread/wait0.internal.h" - -int _futex(int *, int, int, const struct timespec *); +#include "libc/intrin/futex.internal.h" +#include "libc/intrin/pthread.h" +#include "libc/intrin/wait0.internal.h" +#include "libc/linux/futex.h" /** * Blocks until memory location becomes zero. @@ -36,24 +32,12 @@ int _futex(int *, int, int, const struct timespec *); * this operation is to know when it's safe to munmap() a threads stack */ void _wait0(const int *ctid) { - int x, rc; - char buf[12]; + int x; for (;;) { if (!(x = atomic_load_explicit(ctid, memory_order_relaxed))) { break; - } else if (IsLinux() || IsOpenbsd()) { - rc = _futex(ctid, FUTEX_WAIT, x, &(struct timespec){2}); - STRACE("futex(%t, FUTEX_WAIT, %d, {2, 0}) → %s", ctid, x, - (DescribeFutexResult)(buf, rc)); - if (IsOpenbsd() && rc > 0) rc = -rc; - if (!(rc == 0 || // - rc == -EINTR || // - rc == -ETIMEDOUT || // - rc == -EWOULDBLOCK)) { - notpossible; - } } else { - pthread_yield(); + _futex_wait(ctid, x, PTHREAD_PROCESS_SHARED, &(struct timespec){2}); } } if (IsOpenbsd()) { diff --git a/libc/thread/wait0.internal.h b/libc/intrin/wait0.internal.h similarity index 100% rename from libc/thread/wait0.internal.h rename to libc/intrin/wait0.internal.h diff --git a/libc/intrin/wantcrashreports.c b/libc/intrin/wantcrashreports.c deleted file mode 100644 index 8b2d88935..000000000 --- a/libc/intrin/wantcrashreports.c +++ /dev/null @@ -1,21 +0,0 @@ -/*-*- mode:c;indent-tabs-mode:nil;c-basic-offset:2;tab-width:8;coding:utf-8 -*-│ -│vi: set net ft=c ts=2 sts=2 sw=2 fenc=utf-8 :vi│ -╞══════════════════════════════════════════════════════════════════════════════╡ -│ Copyright 2022 Justine Alexandra Roberts Tunney │ -│ │ -│ Permission to use, copy, modify, and/or distribute this software for │ -│ any purpose with or without fee is hereby granted, provided that the │ -│ above copyright notice and this permission notice appear in all copies. │ -│ │ -│ THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL │ -│ WARRANTIES WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED │ -│ WARRANTIES OF MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE │ -│ AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL │ -│ DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR │ -│ PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER │ -│ TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR │ -│ PERFORMANCE OF THIS SOFTWARE. │ -╚─────────────────────────────────────────────────────────────────────────────*/ -#include "libc/log/internal.h" - -bool _wantcrashreports; diff --git a/libc/isystem/limits.h b/libc/isystem/limits.h index 6eb8b5ff6..a6abd1c8b 100644 --- a/libc/isystem/limits.h +++ b/libc/isystem/limits.h @@ -1,6 +1,6 @@ #ifndef LIBC_ISYSTEM_LIMITS_H_ #define LIBC_ISYSTEM_LIMITS_H_ -#include "libc/thread/thread.h" +#include "libc/intrin/pthread.h" #include "libc/limits.h" #include "libc/sysv/consts/_posix.h" #include "libc/sysv/consts/iov.h" diff --git a/libc/isystem/linux/futex.h b/libc/isystem/linux/futex.h deleted file mode 100644 index b07fc1454..000000000 --- a/libc/isystem/linux/futex.h +++ /dev/null @@ -1,5 +0,0 @@ -#ifndef COSMOPOLITAN_LIBC_ISYSTEM_LINUX_FUTEX_H_ -#define COSMOPOLITAN_LIBC_ISYSTEM_LINUX_FUTEX_H_ -#include "libc/sysv/consts/futex.h" -#include "libc/sysv/consts/nr.h" -#endif /* COSMOPOLITAN_LIBC_ISYSTEM_LINUX_FUTEX_H_ */ diff --git a/libc/isystem/pthread.h b/libc/isystem/pthread.h index 77ba04720..9e5e99b40 100644 --- a/libc/isystem/pthread.h +++ b/libc/isystem/pthread.h @@ -1,5 +1,5 @@ #ifndef LIBC_ISYSTEM_PTHREAD_H_ #define LIBC_ISYSTEM_PTHREAD_H_ -#include "libc/thread/thread.h" -#include "libc/thread/thread2.h" +#include "libc/intrin/pthread.h" +#include "libc/intrin/pthread2.h" #endif /* LIBC_ISYSTEM_PTHREAD_H_ */ diff --git a/libc/isystem/semaphore.h b/libc/isystem/semaphore.h deleted file mode 100644 index a672765e1..000000000 --- a/libc/isystem/semaphore.h +++ /dev/null @@ -1,4 +0,0 @@ -#ifndef COSMOPOLITAN_LIBC_ISYSTEM_SEMAPHORE_H_ -#define COSMOPOLITAN_LIBC_ISYSTEM_SEMAPHORE_H_ -#include "libc/calls/semaphore.internal.h" -#endif /* COSMOPOLITAN_LIBC_ISYSTEM_SEMAPHORE_H_ */ diff --git a/libc/isystem/stdio.h b/libc/isystem/stdio.h index cff507487..9e5d61618 100644 --- a/libc/isystem/stdio.h +++ b/libc/isystem/stdio.h @@ -2,7 +2,7 @@ #define LIBC_ISYSTEM_STDIO_H_ #include "libc/calls/calls.h" #include "libc/fmt/fmt.h" -#include "libc/stdio/lock.internal.h" +#include "libc/stdio/lock.h" #include "libc/stdio/stdio.h" #include "libc/stdio/temp.h" #endif diff --git a/libc/isystem/time.h b/libc/isystem/time.h index 64a6b69fa..89449f24c 100644 --- a/libc/isystem/time.h +++ b/libc/isystem/time.h @@ -3,7 +3,6 @@ #include "libc/calls/struct/timespec.h" #include "libc/calls/struct/timeval.h" #include "libc/calls/weirdtypes.h" -#include "libc/sysv/consts/clock.h" #include "libc/sysv/consts/sched.h" #include "libc/time/struct/tm.h" #include "libc/time/time.h" diff --git a/libc/log/backtrace2.c b/libc/log/backtrace2.c index 2459941a4..65b00e55e 100644 --- a/libc/log/backtrace2.c +++ b/libc/log/backtrace2.c @@ -37,6 +37,8 @@ #include "libc/mem/alg.h" #include "libc/mem/bisectcarleft.internal.h" #include "libc/nexgen32e/gc.internal.h" +#include "libc/nexgen32e/gettls.h" +#include "libc/nexgen32e/threaded.h" #include "libc/runtime/gc.internal.h" #include "libc/runtime/runtime.h" #include "libc/runtime/stack.h" @@ -47,7 +49,7 @@ #include "libc/sysv/consts/fileno.h" #include "libc/sysv/consts/o.h" #include "libc/sysv/consts/sig.h" -#include "libc/thread/tls.h" +#include "libc/thread/thread.h" #include "libc/x/x.h" #define kBacktraceMaxFrames 128 @@ -107,7 +109,7 @@ static int PrintBacktraceUsingAddr2line(int fd, const struct StackFrame *bp) { argv[i++] = "-a"; /* filter out w/ shell script wrapper for old versions */ argv[i++] = "-pCife"; argv[i++] = debugbin; - garbage = __tls_enabled ? __get_tls()->tib_garbages : 0; + garbage = __tls_enabled ? ((cthread_t)__get_tls())->garbages : 0; gi = garbage ? garbage->i : 0; for (frame = bp; frame && i < kBacktraceMaxFrames - 1; frame = frame->next) { addr = frame->addr; diff --git a/libc/log/backtrace3.c b/libc/log/backtrace3.c index 2e5b3c82d..eb07cc176 100644 --- a/libc/log/backtrace3.c +++ b/libc/log/backtrace3.c @@ -26,12 +26,14 @@ #include "libc/macros.internal.h" #include "libc/mem/bisectcarleft.internal.h" #include "libc/nexgen32e/gc.internal.h" +#include "libc/nexgen32e/gettls.h" #include "libc/nexgen32e/stackframe.h" +#include "libc/nexgen32e/threaded.h" #include "libc/runtime/memtrack.internal.h" #include "libc/runtime/runtime.h" #include "libc/runtime/symbols.internal.h" #include "libc/str/str.h" -#include "libc/thread/tls.h" +#include "libc/thread/thread.h" #define LIMIT 100 @@ -55,7 +57,7 @@ noinstrument noasan int PrintBacktraceUsingSymbols(int fd, struct Garbages *garbage; const struct StackFrame *frame; if (!bp) bp = __builtin_frame_address(0); - garbage = __tls_enabled ? __get_tls()->tib_garbages : 0; + garbage = __tls_enabled ? ((cthread_t)__get_tls())->garbages : 0; gi = garbage ? garbage->i : 0; for (i = 0, frame = bp; frame; frame = frame->next) { if (++i == LIMIT) { diff --git a/libc/log/internal.h b/libc/log/internal.h index 0d7480f73..51fd64b46 100644 --- a/libc/log/internal.h +++ b/libc/log/internal.h @@ -7,7 +7,6 @@ COSMOPOLITAN_C_START_ extern hidden bool __nocolor; extern hidden int kCrashSigs[8]; -extern hidden bool _wantcrashreports; extern hidden bool g_isrunningundermake; void __start_fatal(const char *, int) hidden; diff --git a/libc/log/leaks.c b/libc/log/leaks.c index 4ec7371f4..2b9d0c70c 100644 --- a/libc/log/leaks.c +++ b/libc/log/leaks.c @@ -16,10 +16,9 @@ │ TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR │ │ PERFORMANCE OF THIS SOFTWARE. │ ╚─────────────────────────────────────────────────────────────────────────────*/ -#include "libc/calls/strace.internal.h" -#include "libc/dce.h" -#include "libc/intrin/asan.internal.h" #include "libc/intrin/bits.h" +#include "libc/calls/strace.internal.h" +#include "libc/intrin/asan.internal.h" #include "libc/intrin/kprintf.h" #include "libc/intrin/lockcmpxchg.h" #include "libc/mem/mem.h" @@ -38,7 +37,7 @@ static bool hasleaks; static noasan void CheckLeak(void *x, void *y, size_t n, void *a) { if (n) { if (IsAsan()) { - if (__asan_get_heap_size(x) && !__asan_is_leaky(x)) { + if (__asan_get_heap_size(x)) { hasleaks = true; } } else { @@ -54,10 +53,9 @@ static noasan void OnMemory(void *x, void *y, size_t n, void *a) { if (i < MAXLEAKS) { ++i; kprintf("%p %,lu bytes [dlmalloc]", x, n); - if (__asan_is_leaky(x)) { - kprintf(" [leaky]"); + if (IsAsan()) { + __asan_print_trace(x); } - __asan_print_trace(x); kprintf("\n"); } else if (i == MAXLEAKS) { ++i; @@ -81,7 +79,6 @@ static noasan bool HasLeaks(void) { */ noasan void CheckForMemoryLeaks(void) { struct mallinfo mi; - if (!IsAsan()) return; // we need traces to exclude leaky if (!_lockcmpxchg(&once, false, true)) { kprintf("CheckForMemoryLeaks() may only be called once\n"); exit(1); diff --git a/libc/log/libfatal.internal.h b/libc/log/libfatal.internal.h index 8ff3934ee..ddf054af6 100644 --- a/libc/log/libfatal.internal.h +++ b/libc/log/libfatal.internal.h @@ -3,6 +3,7 @@ #include "libc/calls/calls.h" #include "libc/dce.h" #include "libc/macros.internal.h" +#include "libc/nexgen32e/bsr.h" #include "libc/nt/runtime.h" #include "libc/sysv/consts/nr.h" #if !(__ASSEMBLER__ + __LINKER__ + 0) @@ -205,7 +206,7 @@ forceinline char *__fixcpy(char p[hasatleast 17], uint64_t x, uint8_t k) { } forceinline char *__hexcpy(char p[hasatleast 17], uint64_t x) { - return __fixcpy(p, x, ROUNDUP(x ? (__builtin_clzll(x) ^ 63) + 1 : 1, 4)); + return __fixcpy(p, x, ROUNDUP(x ? bsrl(x) + 1 : 1, 4)); } forceinline const void *__memchr(const void *s, unsigned char c, size_t n) { diff --git a/libc/log/oncrash.c b/libc/log/oncrash.c index 5f722b23d..8fffe7269 100644 --- a/libc/log/oncrash.c +++ b/libc/log/oncrash.c @@ -35,7 +35,7 @@ #include "libc/log/log.h" #include "libc/macros.internal.h" #include "libc/nexgen32e/stackframe.h" -#include "libc/thread/tls.h" +#include "libc/nexgen32e/threaded.h" #include "libc/runtime/internal.h" #include "libc/runtime/pc.internal.h" #include "libc/runtime/runtime.h" diff --git a/libc/log/printgarbage.c b/libc/log/printgarbage.c index c14d51f51..911a1d986 100644 --- a/libc/log/printgarbage.c +++ b/libc/log/printgarbage.c @@ -20,8 +20,10 @@ #include "libc/intrin/kprintf.h" #include "libc/log/log.h" #include "libc/nexgen32e/gc.internal.h" +#include "libc/nexgen32e/gettls.h" +#include "libc/nexgen32e/threaded.h" #include "libc/stdio/stdio.h" -#include "libc/thread/tls.h" +#include "libc/thread/thread.h" // clang-format off /** @@ -36,7 +38,7 @@ void PrintGarbage(void) { kprintf(" SHADOW STACK @ %p\n", __builtin_frame_address(0)); kprintf("garbage ent. parent frame original ret callback arg \n"); kprintf("------------ ------------ ------------------ ------------------ ------------------\n"); - if ((g = __tls_enabled ? __get_tls()->tib_garbages:0) && g->i) { + if ((g = __tls_enabled ? ((cthread_t)__get_tls())->garbages:0) && g->i) { for (i = g->i; i--;) { symbol = __get_symbol_by_addr(g->p[i].ret); if (symbol) { diff --git a/libc/log/showcrashreports.c b/libc/log/showcrashreports.c index 66615ef89..fafdddf1c 100644 --- a/libc/log/showcrashreports.c +++ b/libc/log/showcrashreports.c @@ -105,7 +105,6 @@ static void FreeSigAltStack(void *p) { void ShowCrashReports(void) { char *sp; struct sigaltstack ss; - _wantcrashreports = true; /* : showcrashreports.c, oncrashthunks.S, oncrash.c */ kCrashSigs[0] = SIGQUIT; /* ctrl+\ aka ctrl+break */ kCrashSigs[1] = SIGFPE; /* 1 / 0 */ @@ -122,7 +121,8 @@ void ShowCrashReports(void) { ss.ss_size = GetStackSize(); // FreeBSD sigaltstack() will EFAULT if we use MAP_STACK here // OpenBSD sigaltstack() auto-applies MAP_STACK to the memory - if ((sp = _mapanon(GetStackSize()))) { + if ((sp = mmap(0, GetStackSize(), PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0)) != MAP_FAILED) { ss.ss_sp = sp; if (!sigaltstack(&ss, &g_oldsigaltstack)) { __cxa_atexit(FreeSigAltStack, ss.ss_sp, 0); diff --git a/libc/log/vflogf.c b/libc/log/vflogf.c index 627889fa4..c8941e48b 100644 --- a/libc/log/vflogf.c +++ b/libc/log/vflogf.c @@ -32,7 +32,7 @@ #include "libc/math.h" #include "libc/nexgen32e/nexgen32e.h" #include "libc/runtime/runtime.h" -#include "libc/stdio/lock.internal.h" +#include "libc/stdio/lock.h" #include "libc/stdio/stdio.h" #include "libc/str/str.h" #include "libc/sysv/consts/fileno.h" diff --git a/libc/mem/gc.c b/libc/mem/gc.c index 96e493c53..3e0dfde84 100644 --- a/libc/mem/gc.c +++ b/libc/mem/gc.c @@ -20,10 +20,12 @@ #include "libc/intrin/likely.h" #include "libc/mem/mem.h" #include "libc/nexgen32e/gc.internal.h" +#include "libc/nexgen32e/gettls.h" #include "libc/nexgen32e/stackframe.h" +#include "libc/nexgen32e/threaded.h" #include "libc/runtime/runtime.h" #include "libc/str/str.h" -#include "libc/thread/tls.h" +#include "libc/thread/thread.h" static inline bool PointerNotOwnedByParentStackFrame(struct StackFrame *frame, struct StackFrame *parent, @@ -34,11 +36,11 @@ static inline bool PointerNotOwnedByParentStackFrame(struct StackFrame *frame, static void TeardownGc(void) { int i; + cthread_t tls; struct Garbages *g; - struct CosmoTib *t; if (__tls_enabled) { - t = __get_tls(); - if ((g = t->tib_garbages)) { + tls = (cthread_t)__get_tls(); + if ((g = tls->garbages)) { // exit() currently doesn't use _gclongjmp() like pthread_exit() // so we need to run the deferred functions manually. while (g->i) { @@ -59,18 +61,18 @@ __attribute__((__constructor__)) static void InitializeGc(void) { // then rewrite caller's return address on stack. static void DeferFunction(struct StackFrame *frame, void *fn, void *arg) { int n2; + cthread_t tls; struct Garbage *p2; struct Garbages *g; - struct CosmoTib *t; - __require_tls(); - t = __get_tls(); - g = t->tib_garbages; + TlsIsRequired(); + tls = (cthread_t)__get_tls(); + g = tls->garbages; if (UNLIKELY(!g)) { if (!(g = malloc(sizeof(struct Garbages)))) notpossible; g->i = 0; g->n = 4; if (!(g->p = malloc(g->n * sizeof(struct Garbage)))) notpossible; - t->tib_garbages = g; + tls->garbages = g; } else if (UNLIKELY(g->i == g->n)) { p2 = g->p; n2 = g->n + (g->n >> 1); diff --git a/libc/nexgen32e/bsr.h b/libc/nexgen32e/bsr.h index c35b731b9..7a4179bdd 100644 --- a/libc/nexgen32e/bsr.h +++ b/libc/nexgen32e/bsr.h @@ -6,9 +6,9 @@ COSMOPOLITAN_C_START_ int bsr(int) pureconst; int bsrl(long) pureconst; int bsrll(long long) pureconst; +int bsr128(uint128_t) pureconst; #if defined(__GNUC__) && defined(__x86_64__) && !defined(__STRICT_ANSI__) -int bsr128(uint128_t) pureconst; #define bsr(u) \ ({ \ unsigned BiTs; \ diff --git a/libc/nexgen32e/gc.S b/libc/nexgen32e/gc.S index d9a41adff..83143e336 100644 --- a/libc/nexgen32e/gc.S +++ b/libc/nexgen32e/gc.S @@ -34,7 +34,7 @@ // @see test/libc/runtime/gc_test.c // @threadsafe __gc: mov %fs:0,%rcx # __get_tls() - mov 0x18(%rcx),%rcx # tls::garbages + mov 0x18(%rcx),%rcx # cthread_t::garbages decl (%rcx) # ++g->i mov (%rcx),%r8d # r8 = g->i mov 8(%rcx),%r9 # r9 = g->p diff --git a/libc/nexgen32e/gclongjmp.S b/libc/nexgen32e/gclongjmp.S index fc98ca623..4bfdd3755 100644 --- a/libc/nexgen32e/gclongjmp.S +++ b/libc/nexgen32e/gclongjmp.S @@ -35,7 +35,7 @@ _gclongjmp: mov %rsp,%rbp .profilable mov %fs:0,%r12 # __get_tls() - mov 0x18(%r12),%r12 # Tls::garbages + mov 0x18(%r12),%r12 # cthread_t::garbages test %r12,%r12 jz 0f movl (%r12),%r13d # garbages.i diff --git a/libc/thread/tls2.h b/libc/nexgen32e/gettls.h similarity index 56% rename from libc/thread/tls2.h rename to libc/nexgen32e/gettls.h index f5ef79147..e9a930989 100644 --- a/libc/thread/tls2.h +++ b/libc/nexgen32e/gettls.h @@ -1,10 +1,23 @@ -#ifndef COSMOPOLITAN_LIBC_THREAD_TLS2_H_ -#define COSMOPOLITAN_LIBC_THREAD_TLS2_H_ +#ifndef COSMOPOLITAN_LIBC_NEXGEN32E_GETTLS_H_ +#define COSMOPOLITAN_LIBC_NEXGEN32E_GETTLS_H_ #include "libc/dce.h" -#include "libc/thread/tls.h" +#include "libc/nexgen32e/threaded.h" #if !(__ASSEMBLER__ + __LINKER__ + 0) COSMOPOLITAN_C_START_ +#if defined(__GNUC__) && defined(__x86_64__) && !defined(__STRICT_ANSI__) +/** + * Returns location of thread information block. + * + * This can't be used in privileged functions. + */ +static noasan inline char *__get_tls(void) { + char *tib; + asm("mov\t%%fs:0,%0" : "=r"(tib) : /* no inputs */ : "memory"); + return tib; +} +#endif /* GNU x86-64 */ + #if defined(__GNUC__) && defined(__x86_64__) && !defined(__STRICT_ANSI__) /** * Returns location of thread information block. @@ -12,7 +25,7 @@ COSMOPOLITAN_C_START_ * This should be favored over __get_tls() for .privileged code that * can't be self-modified by __enable_tls(). */ -static noasan inline struct CosmoTib *__get_tls_privileged(void) { +static noasan inline char *__get_tls_privileged(void) { char *tib, *lin = (char *)0x30; if (IsLinux() || IsFreebsd() || IsNetbsd() || IsOpenbsd() || IsMetal()) { asm("mov\t%%fs:(%1),%0" : "=a"(tib) : "r"(lin) : "memory"); @@ -22,10 +35,10 @@ static noasan inline struct CosmoTib *__get_tls_privileged(void) { tib = *(char **)(tib + 0x1480 + __tls_index * 8); } } - return (struct CosmoTib *)tib; + return tib; } #endif /* GNU x86-64 */ COSMOPOLITAN_C_END_ #endif /* !(__ASSEMBLER__ + __LINKER__ + 0) */ -#endif /* COSMOPOLITAN_LIBC_THREAD_TLS2_H_ */ +#endif /* COSMOPOLITAN_LIBC_NEXGEN32E_GETTLS_H_ */ diff --git a/libc/nexgen32e/threaded.c b/libc/nexgen32e/threaded.c index 6226e1bff..83c56b1a9 100644 --- a/libc/nexgen32e/threaded.c +++ b/libc/nexgen32e/threaded.c @@ -16,7 +16,7 @@ │ TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR │ │ PERFORMANCE OF THIS SOFTWARE. │ ╚─────────────────────────────────────────────────────────────────────────────*/ -#include "libc/thread/tls.h" +#include "libc/nexgen32e/threaded.h" /** * Contains TID of main thread or 0 if threading isn't enabled. diff --git a/libc/nexgen32e/threaded.h b/libc/nexgen32e/threaded.h new file mode 100644 index 000000000..7878eb841 --- /dev/null +++ b/libc/nexgen32e/threaded.h @@ -0,0 +1,16 @@ +#ifndef COSMOPOLITAN_LIBC_NEXGEN32E_THREADED_H_ +#define COSMOPOLITAN_LIBC_NEXGEN32E_THREADED_H_ +#if !(__ASSEMBLER__ + __LINKER__ + 0) +COSMOPOLITAN_C_START_ + +extern int __threaded; +extern bool __tls_enabled; +extern unsigned __tls_index; + +void TlsIsRequired(void); +void *__initialize_tls(char[64]); +void __install_tls(char[64]); + +COSMOPOLITAN_C_END_ +#endif /* !(__ASSEMBLER__ + __LINKER__ + 0) */ +#endif /* COSMOPOLITAN_LIBC_NEXGEN32E_THREADED_H_ */ diff --git a/libc/nt/codegen.sh b/libc/nt/codegen.sh index 35e3f46f0..d1e64d2a5 100644 --- a/libc/nt/codegen.sh +++ b/libc/nt/codegen.sh @@ -1,4 +1,4 @@ -/usr/bin/env echo ' -*-mode:sh;indent-tabs-mode:nil;tab-width:8;coding:utf-8-*-│ +/usr/bin/env echo ' -*- mode:sh; indent-tabs-mode:nil; tab-width:8; coding:utf-8 -*-│ │vi: set net ft=sh ts=2 sts=2 sw=2 fenc=utf-8 :vi│ ╞══════════════════════════════════════════════════════════════════════════════╡ │ Copyright 2020 Justine Alexandra Roberts Tunney │ diff --git a/libc/nt/kernel32/CreateSemaphoreW.s b/libc/nt/kernel32/CreateSemaphoreW.s index febaeb9b6..bcac9a2e9 100644 --- a/libc/nt/kernel32/CreateSemaphoreW.s +++ b/libc/nt/kernel32/CreateSemaphoreW.s @@ -1,12 +1,2 @@ .include "o/libc/nt/codegen.inc" .imp kernel32,__imp_CreateSemaphoreW,CreateSemaphoreW,0 - - .text.windows -CreateSemaphore: - push %rbp - mov %rsp,%rbp - .profilable - mov __imp_CreateSemaphoreW(%rip),%rax - jmp __sysv2nt - .endfn CreateSemaphore,globl - .previous diff --git a/libc/nt/kernel32/LockFileEx.s b/libc/nt/kernel32/LockFileEx.s index f74843927..9aba0a595 100644 --- a/libc/nt/kernel32/LockFileEx.s +++ b/libc/nt/kernel32/LockFileEx.s @@ -2,11 +2,11 @@ .imp kernel32,__imp_LockFileEx,LockFileEx,0 .text.windows -__LockFileEx: +LockFileEx: push %rbp mov %rsp,%rbp .profilable mov __imp_LockFileEx(%rip),%rax jmp __sysv2nt6 - .endfn __LockFileEx,globl + .endfn LockFileEx,globl .previous diff --git a/libc/nt/kernel32/SuspendThread.s b/libc/nt/kernel32/SuspendThread.s index 76654fa04..350b06471 100644 --- a/libc/nt/kernel32/SuspendThread.s +++ b/libc/nt/kernel32/SuspendThread.s @@ -1,15 +1,2 @@ .include "o/libc/nt/codegen.inc" .imp kernel32,__imp_SuspendThread,SuspendThread,0 - - .text.windows -SuspendThread: - push %rbp - mov %rsp,%rbp - .profilable - mov %rdi,%rcx - sub $32,%rsp - call *__imp_SuspendThread(%rip) - leave - ret - .endfn SuspendThread,globl - .previous diff --git a/libc/nt/kernel32/UnlockFileEx.s b/libc/nt/kernel32/UnlockFileEx.s index 0dedd8b8b..577232d60 100644 --- a/libc/nt/kernel32/UnlockFileEx.s +++ b/libc/nt/kernel32/UnlockFileEx.s @@ -2,11 +2,11 @@ .imp kernel32,__imp_UnlockFileEx,UnlockFileEx,0 .text.windows -__UnlockFileEx: +UnlockFileEx: push %rbp mov %rsp,%rbp .profilable mov __imp_UnlockFileEx(%rip),%rax jmp __sysv2nt6 - .endfn __UnlockFileEx,globl + .endfn UnlockFileEx,globl .previous diff --git a/libc/nt/master.sh b/libc/nt/master.sh index 415068d57..29a438991 100755 --- a/libc/nt/master.sh +++ b/libc/nt/master.sh @@ -166,7 +166,7 @@ imp 'CreateMutexEx' CreateMutexExW kernel32 0 imp 'CreatePrivateNamespace' CreatePrivateNamespaceW kernel32 0 imp 'CreateRemoteThread' CreateRemoteThread kernel32 0 imp 'CreateRemoteThreadEx' CreateRemoteThreadEx kernel32 0 -imp 'CreateSemaphore' CreateSemaphoreW kernel32 0 4 +imp 'CreateSemaphore' CreateSemaphoreW kernel32 0 imp 'CreateSemaphoreEx' CreateSemaphoreExW kernel32 0 imp 'CreateSymbolicLinkTransacted' CreateSymbolicLinkTransactedW kernel32 238 imp 'CreateTapePartition' CreateTapePartition kernel32 240 @@ -700,6 +700,7 @@ imp 'LocalUnlock' LocalUnlock kernel32 0 imp 'LocaleNameToLCID' LocaleNameToLCID kernel32 0 imp 'LocateXStateFeature' LocateXStateFeature kernel32 0 imp 'LockFile' LockFile kernel32 0 5 +imp 'LockFileEx' LockFileEx kernel32 0 6 imp 'LockResource' LockResource kernel32 0 1 imp 'MapUserPhysicalPages' MapUserPhysicalPages kernel32 0 imp 'MapUserPhysicalPagesScatter' MapUserPhysicalPagesScatter kernel32 986 @@ -982,7 +983,7 @@ imp 'SleepConditionVariableSRW' SleepConditionVariableSRW kernel32 0 imp 'SleepEx' SleepEx kernel32 0 2 imp 'SortCloseHandle' SortCloseHandle kernel32 1416 imp 'SortGetHandle' SortGetHandle kernel32 1417 -imp 'SuspendThread' SuspendThread kernel32 0 1 +imp 'SuspendThread' SuspendThread kernel32 0 imp 'SwitchToFiber' SwitchToFiber kernel32 0 imp 'SwitchToThread' SwitchToThread kernel32 0 imp 'SystemTimeToFileTime' SystemTimeToFileTime kernel32 0 2 @@ -1023,6 +1024,7 @@ imp 'UTUnRegister' UTUnRegister kernel32 1459 imp 'UmsThreadYield' UmsThreadYield kernel32 1460 imp 'UnhandledExceptionFilter' UnhandledExceptionFilter kernel32 0 imp 'UnlockFile' UnlockFile kernel32 0 5 +imp 'UnlockFileEx' UnlockFileEx kernel32 0 5 imp 'UnmapViewOfFile2' UnmapViewOfFile2 kernel32 0 2 imp 'UnmapViewOfFileEx' UnmapViewOfFileEx kernel32 0 3 imp 'UnregisterApplicationRecoveryCallback' UnregisterApplicationRecoveryCallback kernel32 1466 @@ -1118,7 +1120,6 @@ imp '__FlushViewOfFile' FlushViewOfFile kernel32 0 2 imp '__GenerateConsoleCtrlEvent' GenerateConsoleCtrlEvent kernel32 0 2 imp '__GetExitCodeProcess' GetExitCodeProcess kernel32 0 2 imp '__GetFileAttributes' GetFileAttributesW kernel32 0 1 -imp '__LockFileEx' LockFileEx kernel32 0 6 imp '__MapViewOfFileEx' MapViewOfFileEx kernel32 0 6 imp '__MapViewOfFileExNuma' MapViewOfFileExNuma kernel32 0 7 imp '__MoveFileEx' MoveFileExW kernel32 0 3 @@ -1127,7 +1128,6 @@ imp '__ReOpenFile' ReOpenFile kernel32 0 4 # TODO(jart): 6.2 and highe imp '__RemoveDirectory' RemoveDirectoryW kernel32 0 1 imp '__SetCurrentDirectory' SetCurrentDirectoryW kernel32 0 1 imp '__TerminateProcess' TerminateProcess kernel32 0 2 -imp '__UnlockFileEx' UnlockFileEx kernel32 0 5 imp '__UnmapViewOfFile' UnmapViewOfFile kernel32 0 1 imp '__VirtualProtect' VirtualProtect kernel32 0 4 imp '__WaitForMultipleObjects' WaitForMultipleObjects kernel32 0 4 diff --git a/libc/nt/synchronization.h b/libc/nt/synchronization.h index 5c0fe55eb..cdf052d95 100644 --- a/libc/nt/synchronization.h +++ b/libc/nt/synchronization.h @@ -82,10 +82,6 @@ bool32 SetWaitableTimer(int64_t hTimer, const int64_t *lpDueTimeAsFtOrNegRela, int32_t opt_lPeriodMs, NtTimerapcroutine opt_callback, void *lpArgToCallback, bool32 fUnsleepSystem); -int64_t CreateSemaphore(struct NtSecurityAttributes *opt_lpSemaphoreAttributes, - uint32_t lInitialCount, uint32_t lMaximumCount, - const char16_t *opt_lpName); - int32_t SetEvent(int64_t hEvent); int32_t ResetEvent(int64_t hEvent); int32_t PulseEvent(int64_t hEvent); diff --git a/libc/thread/clone-linux.S b/libc/runtime/clone-linux.S similarity index 100% rename from libc/thread/clone-linux.S rename to libc/runtime/clone-linux.S diff --git a/libc/thread/clone-openbsd.S b/libc/runtime/clone-openbsd.S similarity index 100% rename from libc/thread/clone-openbsd.S rename to libc/runtime/clone-openbsd.S diff --git a/libc/thread/clone.c b/libc/runtime/clone.c similarity index 97% rename from libc/thread/clone.c rename to libc/runtime/clone.c index 37f1bce6b..e869c4f13 100644 --- a/libc/thread/clone.c +++ b/libc/runtime/clone.c @@ -23,8 +23,11 @@ #include "libc/dce.h" #include "libc/errno.h" #include "libc/intrin/asan.internal.h" +#include "libc/intrin/pthread.h" #include "libc/limits.h" #include "libc/macros.internal.h" +#include "libc/nexgen32e/gettls.h" +#include "libc/nexgen32e/threaded.h" #include "libc/nt/runtime.h" #include "libc/nt/thread.h" #include "libc/nt/thunk/msabi.h" @@ -39,8 +42,6 @@ #include "libc/sysv/errfuns.h" #include "libc/thread/freebsd.internal.h" #include "libc/thread/openbsd.internal.h" -#include "libc/thread/thread.h" -#include "libc/thread/tls.h" #include "libc/thread/xnu.internal.h" #define __NR_thr_new 455 @@ -215,7 +216,7 @@ static int CloneXnu(int (*fn)(void *), char *stk, size_t stksz, int flags, wt->ctid = flags & CLONE_CHILD_SETTID ? ctid : &wt->tid; wt->ztid = flags & CLONE_CHILD_CLEARTID ? ctid : &wt->tid; wt->tls = flags & CLONE_SETTLS ? tls : 0; - wt->lock._lock = 1; + wt->lock.lock = 1; if ((rc = bsdthread_create(fn, arg, wt, 0, PTHREAD_START_CUSTOM_XNU)) != -1) { pthread_spin_lock(&wt->lock); rc = wt->tid; @@ -494,6 +495,20 @@ static int CloneLinux(int (*func)(void *arg, int tid), char *stk, size_t stksz, * _Exit1() which terminates the thread. Even though the callback says * it supports a return code, that'll only work on Linux and Windows. * + * The `tls` parameter is for thread-local storage. If you specify this + * then clone() will implicitly rewire libc (e.g. errno) to use TLS: + * + * static char tib[64]; + * __initialize_tls(tib); + * __install_tls(tib); + * + * If you want a main process TLS size that's larger call it manually. + * Once you've done the above and/or started creating your own threads + * you'll be able to access your `tls` thread information block, using + * + * char *p = __get_tls(); + * printf("errno is %d\n", *(int *)(p + 0x3c)); + * * This function follows the same ABI convention as the Linux userspace * libraries, with a few small changes. The varargs has been removed to * help prevent broken code, and the stack size and tls size parameters diff --git a/libc/intrin/leaky.S b/libc/runtime/construct.S similarity index 79% rename from libc/intrin/leaky.S rename to libc/runtime/construct.S index d79d752d2..953e8b4d1 100644 --- a/libc/intrin/leaky.S +++ b/libc/runtime/construct.S @@ -1,7 +1,7 @@ /*-*- mode:unix-assembly; indent-tabs-mode:t; tab-width:8; coding:utf-8 -*-│ │vi: set et ft=asm ts=8 tw=8 fenc=utf-8 :vi│ ╞══════════════════════════════════════════════════════════════════════════════╡ -│ Copyright 2022 Justine Alexandra Roberts Tunney │ +│ Copyright 2020 Justine Alexandra Roberts Tunney │ │ │ │ Permission to use, copy, modify, and/or distribute this software for │ │ any purpose with or without fee is hereby granted, provided that the │ @@ -16,24 +16,19 @@ │ TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR │ │ PERFORMANCE OF THIS SOFTWARE. │ ╚─────────────────────────────────────────────────────────────────────────────*/ +#include "libc/runtime/internal.h" #include "libc/macros.internal.h" +.text.startup -// Decentralized section for leaky functions. - .section .piro.relo.sort.leaky.1,"aw",@nobits - .type _leaky_start,@object - .type _leaky_end,@object - .globl _leaky_start,_leaky_end - .hidden _leaky_start,_leaky_end - .byte 0 - .align __SIZEOF_POINTER__ - .underrun -_leaky_start: - .previous/* - ... - decentralized content - ... - */.section .piro.relo.sort.leaky.3,"aw",@nobits -_leaky_end: - .quad 0 - .overrun - .previous +// Calls global initialization functions. +// +// @param r12 is argc +// @param r13 is argv +// @param r14 is environ +// @param r15 is auxv +_construct: + push %rbp + mov %rsp,%rbp + pop %rbp + ret + .endfn _construct,globl diff --git a/libc/runtime/enable_threads.c b/libc/runtime/enable_threads.c index 47a17ee3c..08ededed8 100644 --- a/libc/runtime/enable_threads.c +++ b/libc/runtime/enable_threads.c @@ -19,7 +19,7 @@ #include "libc/assert.h" #include "libc/calls/calls.h" #include "libc/calls/strace.internal.h" -#include "libc/thread/tls.h" +#include "libc/nexgen32e/threaded.h" #include "libc/runtime/runtime.h" extern int __threadcalls_end[]; diff --git a/libc/runtime/enable_tls.c b/libc/runtime/enable_tls.c index 88b6d6e53..b6916d131 100644 --- a/libc/runtime/enable_tls.c +++ b/libc/runtime/enable_tls.c @@ -18,23 +18,23 @@ ╚─────────────────────────────────────────────────────────────────────────────*/ #include "libc/assert.h" #include "libc/calls/calls.h" +#include "libc/calls/strace.internal.h" #include "libc/calls/syscall-sysv.internal.h" #include "libc/dce.h" #include "libc/errno.h" -#include "libc/intrin/asan.internal.h" -#include "libc/intrin/asancodes.h" #include "libc/intrin/bits.h" #include "libc/intrin/weaken.h" #include "libc/log/libfatal.internal.h" #include "libc/macros.internal.h" #include "libc/nexgen32e/msr.h" +#include "libc/nexgen32e/threaded.h" #include "libc/nt/thread.h" #include "libc/runtime/internal.h" #include "libc/runtime/runtime.h" #include "libc/stdalign.internal.h" #include "libc/str/str.h" #include "libc/sysv/consts/nrlinux.h" -#include "libc/thread/tls.h" +#include "libc/thread/thread.h" #include "third_party/xed/x86.h" #define __NR_sysarch 0x000000a5 // freebsd+netbsd @@ -49,7 +49,7 @@ #define _TLSZ ((intptr_t)_tls_size) #define _TLDZ ((intptr_t)_tdata_size) -#define _TIBZ sizeof(struct CosmoTib) +#define _TIBZ sizeof(struct cthread_descriptor_t) int sys_enable_tls(); @@ -59,7 +59,7 @@ __msabi extern typeof(TlsAlloc) *const __imp_TlsAlloc; extern unsigned char __tls_mov_nt_rax[]; extern unsigned char __tls_add_nt_rax[]; -_Alignas(TLS_ALIGNMENT) static char __static_tls[5008]; +_Alignas(long) static char __static_tls[5008]; /** * Enables thread local storage for main process. @@ -98,13 +98,14 @@ _Alignas(TLS_ALIGNMENT) static char __static_tls[5008]; */ privileged void __enable_tls(void) { size_t siz; - struct CosmoTib *tib; + cthread_t tib; char *mem, *tls; siz = ROUNDUP(_TLSZ + _TIBZ, alignof(__static_tls)); if (siz <= sizeof(__static_tls)) { // if tls requirement is small then use the static tls block // which helps avoid a system call for appes with little tls // this is crucial to keeping life.com 16 kilobytes in size! + _Static_assert(alignof(__static_tls) >= alignof(cthread_t)); mem = __static_tls; } else { // if this binary needs a hefty tls block then we'll bank on @@ -116,23 +117,17 @@ privileged void __enable_tls(void) { mem = weaken(_mapanon)(siz); assert(mem); } - if (IsAsan()) { - // poison the space between .tdata and .tbss - __asan_poison(mem + (intptr_t)_tdata_size, - (intptr_t)_tbss_offset - (intptr_t)_tdata_size, - kAsanProtected); - } - tib = (struct CosmoTib *)(mem + siz - _TIBZ); + tib = (cthread_t)(mem + siz - _TIBZ); tls = mem + siz - _TIBZ - _TLSZ; - tib->tib_self = tib; - tib->tib_self2 = tib; - tib->tib_errno = __errno; + tib->self = tib; + tib->self2 = tib; + tib->err = __errno; if (IsLinux()) { // gnu/systemd guarantees pid==tid for the main thread so we can // avoid issuing a superfluous system call at startup in program - tib->tib_tid = __pid; + tib->tid = __pid; } else { - tib->tib_tid = sys_gettid(); + tib->tid = sys_gettid(); } __repmovsb(tls, _tdata_start, _TLDZ); diff --git a/libc/runtime/fork-nt.c b/libc/runtime/fork-nt.c index 73ebcb2e6..a80a6a75b 100644 --- a/libc/runtime/fork-nt.c +++ b/libc/runtime/fork-nt.c @@ -29,7 +29,7 @@ #include "libc/mem/alloca.h" #include "libc/mem/mem.h" #include "libc/nexgen32e/nt2sysv.h" -#include "libc/thread/tls.h" +#include "libc/nexgen32e/threaded.h" #include "libc/nt/console.h" #include "libc/nt/createfile.h" #include "libc/nt/enum/accessmask.h" @@ -226,9 +226,9 @@ textwindows void WinMainForked(void) { // rewrap the stdin named pipe hack // since the handles closed on fork struct Fds *fds = VEIL("r", &g_fds); - fds->p[0].handle = GetStdHandle(kNtStdInputHandle); - fds->p[1].handle = GetStdHandle(kNtStdOutputHandle); - fds->p[2].handle = GetStdHandle(kNtStdErrorHandle); + fds->p[0].handle = fds->__init_p[0].handle = GetStdHandle(kNtStdInputHandle); + fds->p[1].handle = fds->__init_p[1].handle = GetStdHandle(kNtStdOutputHandle); + fds->p[2].handle = fds->__init_p[2].handle = GetStdHandle(kNtStdErrorHandle); // untrack children of parent since we specify with both // CreateProcess() and CreateThread() as non-inheritable diff --git a/libc/runtime/fork.c b/libc/runtime/fork.c index 1d50b3fc1..e717e421e 100644 --- a/libc/runtime/fork.c +++ b/libc/runtime/fork.c @@ -22,10 +22,11 @@ #include "libc/calls/syscall-nt.internal.h" #include "libc/calls/syscall-sysv.internal.h" #include "libc/dce.h" +#include "libc/nexgen32e/gettls.h" +#include "libc/nexgen32e/threaded.h" #include "libc/nt/process.h" #include "libc/runtime/internal.h" #include "libc/sysv/consts/sig.h" -#include "libc/thread/tls.h" /** * Creates new process. @@ -60,7 +61,7 @@ int fork(void) { parent = __pid; __pid = dx; if (__tls_enabled) { - __get_tls()->tib_tid = IsLinux() ? dx : sys_gettid(); + *(int *)(__get_tls() + 0x38) = IsLinux() ? dx : sys_gettid(); } STRACE("fork() → 0 (child of %d)", parent); sigprocmask(SIG_SETMASK, &old, 0); diff --git a/libc/runtime/ftracer.c b/libc/runtime/ftracer.c index 9e3ae1d7d..44d7a611d 100644 --- a/libc/runtime/ftracer.c +++ b/libc/runtime/ftracer.c @@ -21,14 +21,15 @@ #include "libc/fmt/itoa.h" #include "libc/intrin/cmpxchg.h" #include "libc/intrin/kprintf.h" -#include "libc/intrin/nopl.internal.h" +#include "libc/intrin/nopl.h" #include "libc/macros.internal.h" +#include "libc/nexgen32e/gettls.h" #include "libc/nexgen32e/stackframe.h" +#include "libc/nexgen32e/threaded.h" #include "libc/runtime/internal.h" #include "libc/runtime/stack.h" #include "libc/runtime/symbols.internal.h" -#include "libc/thread/tls.h" -#include "libc/thread/tls2.h" +#include "libc/thread/thread.h" #define MAX_NESTING 512 @@ -43,7 +44,7 @@ void ftrace_hook(void); static int g_stackdigs; -static struct CosmoFtrace g_ftrace; +static struct Ftrace g_ftrace; static privileged inline int GetNestingLevelImpl(struct StackFrame *frame) { int nesting = -2; @@ -54,12 +55,12 @@ static privileged inline int GetNestingLevelImpl(struct StackFrame *frame) { return MAX(0, nesting); } -static privileged inline int GetNestingLevel(struct CosmoFtrace *ft, +static privileged inline int GetNestingLevel(struct Ftrace *ft, struct StackFrame *sf) { int nesting; nesting = GetNestingLevelImpl(sf); - if (nesting < ft->ft_skew) ft->ft_skew = nesting; - nesting -= ft->ft_skew; + if (nesting < ft->skew) ft->skew = nesting; + nesting -= ft->skew; return MIN(MAX_NESTING, nesting); } @@ -72,27 +73,27 @@ static privileged inline int GetNestingLevel(struct CosmoFtrace *ft, */ privileged void ftracer(void) { long stackuse; - struct CosmoFtrace *ft; + struct Ftrace *ft; struct StackFrame *sf; if (__tls_enabled) { - ft = &__get_tls_privileged()->tib_ftrace; + ft = (struct Ftrace *)(__get_tls_privileged() + 0x08); } else { ft = &g_ftrace; } - if (_cmpxchg(&ft->ft_once, false, true)) { - ft->ft_lastaddr = -1; - ft->ft_skew = GetNestingLevelImpl(__builtin_frame_address(0)); + if (_cmpxchg(&ft->once, false, true)) { + ft->lastaddr = -1; + ft->skew = GetNestingLevelImpl(__builtin_frame_address(0)); } - if (_cmpxchg(&ft->ft_noreentry, false, true)) { + if (_cmpxchg(&ft->noreentry, false, true)) { sf = __builtin_frame_address(0); sf = sf->next; - if (sf->addr != ft->ft_lastaddr) { + if (sf->addr != ft->lastaddr) { stackuse = GetStackAddr() + GetStackSize() - (intptr_t)sf; kprintf("%rFUN %6P %'13T %'*ld %*s%t\n", g_stackdigs, stackuse, GetNestingLevel(ft, sf) * 2, "", sf->addr); - ft->ft_lastaddr = sf->addr; + ft->lastaddr = sf->addr; } - ft->ft_noreentry = false; + ft->noreentry = false; } } diff --git a/libc/runtime/getsymboltable.c b/libc/runtime/getsymboltable.c index 44c897b14..742504416 100644 --- a/libc/runtime/getsymboltable.c +++ b/libc/runtime/getsymboltable.c @@ -20,7 +20,7 @@ #include "libc/calls/strace.internal.h" #include "libc/intrin/bits.h" #include "libc/intrin/promises.internal.h" -#include "libc/thread/thread.h" +#include "libc/intrin/pthread.h" #include "libc/intrin/weaken.h" #include "libc/macros.internal.h" #include "libc/runtime/internal.h" diff --git a/libc/runtime/hook.greg.c b/libc/runtime/hook.greg.c index 24d7de0a3..c68cbb904 100644 --- a/libc/runtime/hook.greg.c +++ b/libc/runtime/hook.greg.c @@ -16,9 +16,16 @@ │ TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR │ │ PERFORMANCE OF THIS SOFTWARE. │ ╚─────────────────────────────────────────────────────────────────────────────*/ +#include "libc/calls/calls.h" #include "libc/calls/struct/sigset.h" +#include "libc/dce.h" +#include "libc/errno.h" +#include "libc/intrin/bits.h" +#include "libc/log/libfatal.internal.h" #include "libc/runtime/runtime.h" #include "libc/runtime/symbols.internal.h" +#include "libc/str/str.h" +#include "libc/sysv/consts/prot.h" /** * Rewrites code in memory to hook function calls. diff --git a/libc/runtime/internal.h b/libc/runtime/internal.h index 2b0e25534..dd4e9877b 100644 --- a/libc/runtime/internal.h +++ b/libc/runtime/internal.h @@ -23,8 +23,6 @@ extern unsigned char _tdata_end[]; extern unsigned char _tdata_size[]; extern unsigned char _tbss_start[]; extern unsigned char _tbss_end[]; -extern unsigned char _tbss_size[]; -extern unsigned char _tbss_offset[]; extern unsigned char _tls_size[]; extern unsigned char _tls_content[]; diff --git a/libc/runtime/memtrack.internal.h b/libc/runtime/memtrack.internal.h index ff2488f0a..746602693 100644 --- a/libc/runtime/memtrack.internal.h +++ b/libc/runtime/memtrack.internal.h @@ -3,9 +3,9 @@ #include "libc/assert.h" #include "libc/intrin/midpoint.h" #include "libc/dce.h" -#include "libc/intrin/nopl.internal.h" +#include "libc/intrin/nopl.h" #include "libc/macros.internal.h" -#include "libc/thread/tls.h" +#include "libc/nexgen32e/threaded.h" #include "libc/nt/version.h" #include "libc/runtime/stack.h" #include "libc/sysv/consts/ss.h" diff --git a/libc/runtime/memtrack64.txt b/libc/runtime/memtrack64.txt index a16347005..2f5356616 100644 --- a/libc/runtime/memtrack64.txt +++ b/libc/runtime/memtrack64.txt @@ -1808,9 +1808,9 @@ 6f900000-6f9fffff 64gb free 6fa00000-6fafffff 64gb free 6fb00000-6fbfffff 64gb free -6fc00004-6fcfffff 64gb nsync +6fc00000-6fcfffff 64gb free 6fd00000-6fdfffff 64gb zipos -6fe00004-6feffffc 64gb g_fds +6fe00000-6fefffff 64gb g_fds 6ff00000-6ffffffd 64gb free 6ffffffe-6fffffff 128kb winargs diff --git a/libc/runtime/mman.internal.h b/libc/runtime/mman.internal.h index 8c582a52d..9863579c1 100644 --- a/libc/runtime/mman.internal.h +++ b/libc/runtime/mman.internal.h @@ -7,16 +7,16 @@ COSMOPOLITAN_C_START_ struct mman { int64_t pdp; /* 0x0500 */ int32_t pdpi; /* 0x0508 */ - int32_t e820n; /* 0x050c */ + int32_t e820n; /* 0x050a */ struct SmapEntry e820[256]; /* 0x0510 */ - char pc_drive_base_table[11]; /* 0x1d10 */ - unsigned char pc_drive_type; /* 0x1d1b */ - unsigned char pc_drive_last_sector; /* 0x1d1c */ - unsigned short pc_drive_last_cylinder; /* 0x1d1d */ - unsigned char pc_drives_attached; /* 0x1d1f */ - unsigned char pc_drive_last_head; /* 0x1d20 */ - unsigned char pc_drive; /* 0x1d21 */ - char bad_idt[6]; /* 0x1d22 */ + char pc_drive_base_table[11]; /* 0x1510 */ + unsigned char pc_drive_type; /* 0x151b */ + unsigned char pc_drive_last_sector; /* 0x151c */ + unsigned short pc_drive_last_cylinder; /* 0x151d */ + unsigned char pc_drives_attached; /* 0x151f */ + unsigned char pc_drive_last_head; /* 0x1520 */ + unsigned char pc_drive; /* 0x1521 */ + char bad_idt[6]; /* 0x1522 */ }; COSMOPOLITAN_C_END_ diff --git a/libc/runtime/mmap.c b/libc/runtime/mmap.c index 71efc97da..4bdfeaaf9 100644 --- a/libc/runtime/mmap.c +++ b/libc/runtime/mmap.c @@ -28,7 +28,7 @@ #include "libc/intrin/bits.h" #include "libc/intrin/describeflags.internal.h" #include "libc/intrin/likely.h" -#include "libc/thread/thread.h" +#include "libc/intrin/pthread.h" #include "libc/intrin/safemacros.internal.h" #include "libc/intrin/weaken.h" #include "libc/limits.h" diff --git a/libc/intrin/printmemoryintervals.c b/libc/runtime/printmemoryintervals.c similarity index 100% rename from libc/intrin/printmemoryintervals.c rename to libc/runtime/printmemoryintervals.c diff --git a/libc/thread/clone-nt.S b/libc/runtime/winthreadlaunch.S similarity index 100% rename from libc/thread/clone-nt.S rename to libc/runtime/winthreadlaunch.S diff --git a/libc/sock/sendfile.c b/libc/sock/sendfile.c index 4908c2f23..2a1350a33 100644 --- a/libc/sock/sendfile.c +++ b/libc/sock/sendfile.c @@ -23,7 +23,6 @@ #include "libc/calls/syscall-sysv.internal.h" #include "libc/calls/syscall_support-nt.internal.h" #include "libc/dce.h" -#include "libc/intrin/asan.internal.h" #include "libc/intrin/safemacros.internal.h" #include "libc/nt/enum/wait.h" #include "libc/nt/errors.h" @@ -107,13 +106,10 @@ static ssize_t sendfile_linux2bsd(int outfd, int infd, if (IsFreebsd()) { rc = sys_sendfile_freebsd(infd, outfd, offset, uptobytes, 0, &sbytes, 0); } else { - sbytes = uptobytes; rc = sys_sendfile_xnu(infd, outfd, offset, &sbytes, 0, 0); } if (rc != -1) { - if (inout_opt_inoffset) { - *inout_opt_inoffset += sbytes; - } + if (inout_opt_inoffset) *inout_opt_inoffset += sbytes; return sbytes; } else { return -1; @@ -135,25 +131,15 @@ static ssize_t sendfile_linux2bsd(int outfd, int infd, */ ssize_t sendfile(int outfd, int infd, int64_t *inout_opt_inoffset, size_t uptobytes) { - int rc; - if (!uptobytes) { - rc = einval(); - } else if (IsAsan() && inout_opt_inoffset && - !__asan_is_valid(inout_opt_inoffset, - sizeof(*inout_opt_inoffset))) { - rc = efault(); - } else if (uptobytes > 0x7ffffffe /* Microsoft's off-by-one */) { - rc = eoverflow(); - } else if (IsLinux()) { - rc = sys_sendfile(outfd, infd, inout_opt_inoffset, uptobytes); + if (!uptobytes) return einval(); + if (uptobytes > 0x7ffffffe /* Microsoft's off-by-one */) return eoverflow(); + if (IsLinux()) { + return sys_sendfile(outfd, infd, inout_opt_inoffset, uptobytes); } else if (IsFreebsd() || IsXnu()) { - rc = sendfile_linux2bsd(outfd, infd, inout_opt_inoffset, uptobytes); + return sendfile_linux2bsd(outfd, infd, inout_opt_inoffset, uptobytes); } else if (IsWindows()) { - rc = sendfile_linux2nt(outfd, infd, inout_opt_inoffset, uptobytes); + return sendfile_linux2nt(outfd, infd, inout_opt_inoffset, uptobytes); } else { - rc = copyfd(infd, inout_opt_inoffset, outfd, NULL, uptobytes, 0); + return copyfd(infd, inout_opt_inoffset, outfd, NULL, uptobytes, 0); } - STRACE("sendfile(%d, %d, %p, %'zu) → %ld% m", outfd, infd, inout_opt_inoffset, - uptobytes, rc); - return rc; } diff --git a/libc/stdio/clearerr.c b/libc/stdio/clearerr.c index 1ff0ae2b6..723847951 100644 --- a/libc/stdio/clearerr.c +++ b/libc/stdio/clearerr.c @@ -16,7 +16,7 @@ │ TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR │ │ PERFORMANCE OF THIS SOFTWARE. │ ╚─────────────────────────────────────────────────────────────────────────────*/ -#include "libc/stdio/lock.internal.h" +#include "libc/stdio/lock.h" #include "libc/stdio/stdio.h" /** diff --git a/libc/stdio/dirstream.c b/libc/stdio/dirstream.c index e382cd254..6a62df49c 100644 --- a/libc/stdio/dirstream.c +++ b/libc/stdio/dirstream.c @@ -25,8 +25,8 @@ #include "libc/calls/syscall_support-nt.internal.h" #include "libc/errno.h" #include "libc/intrin/asan.internal.h" -#include "libc/intrin/nopl.internal.h" -#include "libc/thread/thread.h" +#include "libc/intrin/nopl.h" +#include "libc/intrin/pthread.h" #include "libc/intrin/weaken.h" #include "libc/mem/mem.h" #include "libc/nt/enum/fileflagandattributes.h" diff --git a/libc/stdio/ecvt.c b/libc/stdio/ecvt.c index 2eba91728..61f4455d9 100644 --- a/libc/stdio/ecvt.c +++ b/libc/stdio/ecvt.c @@ -24,7 +24,6 @@ ╚─────────────────────────────────────────────────────────────────────────────*/ #include "libc/fmt/fmt.h" #include "libc/mem/mem.h" -#include "libc/runtime/runtime.h" #include "libc/str/str.h" #include "third_party/gdtoa/gdtoa.h" @@ -34,24 +33,10 @@ Copyright (c) 2002, 2006, 2010 Todd C. Miller \""); asm(".include \"libc/disclaimer.inc\""); // clang-format off -static char *s; - -static void -__cvt_atexit(void) -{ - free(s); - s = 0; -} - -static void __attribute__((__constructor__)) -__cvt_init(void) -{ - atexit(__cvt_atexit); -} - static char * __cvt(double value, int ndigit, int *decpt, int *sign, int fmode, int pad) { + static char *s; char *p, *rve, c; size_t siz; diff --git a/libc/stdio/fdopen.c b/libc/stdio/fdopen.c index de8875d62..4a25238cd 100644 --- a/libc/stdio/fdopen.c +++ b/libc/stdio/fdopen.c @@ -17,12 +17,12 @@ │ PERFORMANCE OF THIS SOFTWARE. │ ╚─────────────────────────────────────────────────────────────────────────────*/ #include "libc/calls/calls.h" +#include "libc/intrin/pthread.h" #include "libc/mem/mem.h" #include "libc/stdio/internal.h" #include "libc/stdio/stdio.h" #include "libc/sysv/consts/o.h" #include "libc/sysv/errfuns.h" -#include "libc/thread/thread.h" /** * Allocates stream object for already-opened file descriptor. @@ -38,7 +38,7 @@ FILE *fdopen(int fd, const char *mode) { f->fd = fd; f->bufmode = ischardev(fd) ? _IOLBF : _IOFBF; f->iomode = fopenflags(mode); - f->lock._type = PTHREAD_MUTEX_RECURSIVE; + f->lock.type = PTHREAD_MUTEX_RECURSIVE; f->size = BUFSIZ; if ((f->buf = malloc(f->size))) { if ((f->iomode & O_ACCMODE) != O_RDONLY) { diff --git a/libc/stdio/feof.c b/libc/stdio/feof.c index ac225d98a..e115f9715 100644 --- a/libc/stdio/feof.c +++ b/libc/stdio/feof.c @@ -16,7 +16,7 @@ │ TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR │ │ PERFORMANCE OF THIS SOFTWARE. │ ╚─────────────────────────────────────────────────────────────────────────────*/ -#include "libc/stdio/lock.internal.h" +#include "libc/stdio/lock.h" #include "libc/stdio/stdio.h" /** diff --git a/libc/stdio/ferror.c b/libc/stdio/ferror.c index 7b8905d0f..22b46d0c0 100644 --- a/libc/stdio/ferror.c +++ b/libc/stdio/ferror.c @@ -16,7 +16,7 @@ │ TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR │ │ PERFORMANCE OF THIS SOFTWARE. │ ╚─────────────────────────────────────────────────────────────────────────────*/ -#include "libc/stdio/lock.internal.h" +#include "libc/stdio/lock.h" #include "libc/stdio/stdio.h" /** diff --git a/libc/stdio/fflush.c b/libc/stdio/fflush.c index 90da7fa6b..8f454dde7 100644 --- a/libc/stdio/fflush.c +++ b/libc/stdio/fflush.c @@ -16,7 +16,7 @@ │ TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR │ │ PERFORMANCE OF THIS SOFTWARE. │ ╚─────────────────────────────────────────────────────────────────────────────*/ -#include "libc/stdio/lock.internal.h" +#include "libc/stdio/lock.h" #include "libc/stdio/stdio.h" /** diff --git a/libc/stdio/fflush.internal.h b/libc/stdio/fflush.internal.h index cc3343d3a..a00e504fb 100644 --- a/libc/stdio/fflush.internal.h +++ b/libc/stdio/fflush.internal.h @@ -1,7 +1,7 @@ #ifndef COSMOPOLITAN_LIBC_STDIO_FFLUSH_H_ #define COSMOPOLITAN_LIBC_STDIO_FFLUSH_H_ -#include "libc/intrin/nopl.internal.h" -#include "libc/thread/thread.h" +#include "libc/intrin/nopl.h" +#include "libc/intrin/pthread.h" #include "libc/stdio/stdio.h" #if !(__ASSEMBLER__ + __LINKER__ + 0) COSMOPOLITAN_C_START_ diff --git a/libc/stdio/fflush_unlocked.c b/libc/stdio/fflush_unlocked.c index a00ebe969..0746536be 100644 --- a/libc/stdio/fflush_unlocked.c +++ b/libc/stdio/fflush_unlocked.c @@ -19,7 +19,7 @@ #include "libc/calls/calls.h" #include "libc/errno.h" #include "libc/intrin/bits.h" -#include "libc/thread/thread.h" +#include "libc/intrin/pthread.h" #include "libc/intrin/pushpop.h" #include "libc/macros.internal.h" #include "libc/mem/arraylist.internal.h" diff --git a/libc/stdio/fgetc.c b/libc/stdio/fgetc.c index 8dda15ace..b04afd66c 100644 --- a/libc/stdio/fgetc.c +++ b/libc/stdio/fgetc.c @@ -16,7 +16,7 @@ │ TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR │ │ PERFORMANCE OF THIS SOFTWARE. │ ╚─────────────────────────────────────────────────────────────────────────────*/ -#include "libc/stdio/lock.internal.h" +#include "libc/stdio/lock.h" #include "libc/stdio/stdio.h" /** diff --git a/libc/stdio/fgetln.c b/libc/stdio/fgetln.c index 04f8edfca..a131e9dfc 100644 --- a/libc/stdio/fgetln.c +++ b/libc/stdio/fgetln.c @@ -16,7 +16,7 @@ │ TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR │ │ PERFORMANCE OF THIS SOFTWARE. │ ╚─────────────────────────────────────────────────────────────────────────────*/ -#include "libc/stdio/lock.internal.h" +#include "libc/stdio/lock.h" #include "libc/stdio/stdio.h" /** diff --git a/libc/stdio/fgets.c b/libc/stdio/fgets.c index 121260ee9..2807ef537 100644 --- a/libc/stdio/fgets.c +++ b/libc/stdio/fgets.c @@ -16,7 +16,7 @@ │ TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR │ │ PERFORMANCE OF THIS SOFTWARE. │ ╚─────────────────────────────────────────────────────────────────────────────*/ -#include "libc/stdio/lock.internal.h" +#include "libc/stdio/lock.h" #include "libc/stdio/stdio.h" /** diff --git a/libc/stdio/fgetwc.c b/libc/stdio/fgetwc.c index aecf07615..d7e1e41f8 100644 --- a/libc/stdio/fgetwc.c +++ b/libc/stdio/fgetwc.c @@ -16,7 +16,7 @@ │ TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR │ │ PERFORMANCE OF THIS SOFTWARE. │ ╚─────────────────────────────────────────────────────────────────────────────*/ -#include "libc/stdio/lock.internal.h" +#include "libc/stdio/lock.h" #include "libc/stdio/stdio.h" /** diff --git a/libc/stdio/fgetws.c b/libc/stdio/fgetws.c index f116a51f5..3c96e30d9 100644 --- a/libc/stdio/fgetws.c +++ b/libc/stdio/fgetws.c @@ -16,7 +16,7 @@ │ TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR │ │ PERFORMANCE OF THIS SOFTWARE. │ ╚─────────────────────────────────────────────────────────────────────────────*/ -#include "libc/stdio/lock.internal.h" +#include "libc/stdio/lock.h" #include "libc/stdio/stdio.h" /** diff --git a/libc/stdio/fileno.c b/libc/stdio/fileno.c index 696e7f7a2..e739d1cb5 100644 --- a/libc/stdio/fileno.c +++ b/libc/stdio/fileno.c @@ -16,7 +16,7 @@ │ TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR │ │ PERFORMANCE OF THIS SOFTWARE. │ ╚─────────────────────────────────────────────────────────────────────────────*/ -#include "libc/stdio/lock.internal.h" +#include "libc/stdio/lock.h" #include "libc/stdio/stdio.h" #include "libc/sysv/errfuns.h" diff --git a/libc/stdio/flockfile.c b/libc/stdio/flockfile.c index 36ae8a3e5..e1a3872ca 100644 --- a/libc/stdio/flockfile.c +++ b/libc/stdio/flockfile.c @@ -16,7 +16,7 @@ │ TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR │ │ PERFORMANCE OF THIS SOFTWARE. │ ╚─────────────────────────────────────────────────────────────────────────────*/ -#include "libc/stdio/lock.internal.h" +#include "libc/stdio/lock.h" #include "libc/stdio/stdio.h" /** diff --git a/libc/stdio/flushlbf.c b/libc/stdio/flushlbf.c index a254e3c3c..69bd5e4c9 100644 --- a/libc/stdio/flushlbf.c +++ b/libc/stdio/flushlbf.c @@ -17,9 +17,9 @@ │ PERFORMANCE OF THIS SOFTWARE. │ ╚─────────────────────────────────────────────────────────────────────────────*/ #include "libc/calls/calls.h" -#include "libc/thread/thread.h" +#include "libc/intrin/pthread.h" #include "libc/stdio/fflush.internal.h" -#include "libc/stdio/lock.internal.h" +#include "libc/stdio/lock.h" #include "libc/stdio/stdio.h" #include "libc/stdio/stdio_ext.h" diff --git a/libc/stdio/fmemopen.c b/libc/stdio/fmemopen.c index 3183982a3..28c12cd74 100644 --- a/libc/stdio/fmemopen.c +++ b/libc/stdio/fmemopen.c @@ -16,12 +16,12 @@ │ TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR │ │ PERFORMANCE OF THIS SOFTWARE. │ ╚─────────────────────────────────────────────────────────────────────────────*/ +#include "libc/intrin/pthread.h" #include "libc/mem/mem.h" #include "libc/stdio/stdio.h" #include "libc/str/str.h" #include "libc/sysv/consts/o.h" #include "libc/sysv/errfuns.h" -#include "libc/thread/thread.h" /** * Opens buffer as stream. @@ -54,7 +54,7 @@ FILE *fmemopen(void *buf, size_t size, const char *mode) { f->end = size; f->size = size; f->iomode = fopenflags(mode); - f->lock._type = PTHREAD_MUTEX_RECURSIVE; + f->lock.type = PTHREAD_MUTEX_RECURSIVE; if (f->iomode & O_APPEND) { if ((p = memchr(buf, '\0', size))) { f->beg = p - (char *)buf; diff --git a/libc/stdio/fprintf.c b/libc/stdio/fprintf.c index fd512ca11..796394dc7 100644 --- a/libc/stdio/fprintf.c +++ b/libc/stdio/fprintf.c @@ -16,7 +16,7 @@ │ TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR │ │ PERFORMANCE OF THIS SOFTWARE. │ ╚─────────────────────────────────────────────────────────────────────────────*/ -#include "libc/stdio/lock.internal.h" +#include "libc/stdio/lock.h" #include "libc/stdio/stdio.h" /** diff --git a/libc/stdio/fputc.c b/libc/stdio/fputc.c index 28b8a3b72..faf8af218 100644 --- a/libc/stdio/fputc.c +++ b/libc/stdio/fputc.c @@ -16,7 +16,7 @@ │ TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR │ │ PERFORMANCE OF THIS SOFTWARE. │ ╚─────────────────────────────────────────────────────────────────────────────*/ -#include "libc/stdio/lock.internal.h" +#include "libc/stdio/lock.h" #include "libc/stdio/stdio.h" /** diff --git a/libc/stdio/fputs.c b/libc/stdio/fputs.c index f7adc65d9..85bf483e6 100644 --- a/libc/stdio/fputs.c +++ b/libc/stdio/fputs.c @@ -16,7 +16,7 @@ │ TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR │ │ PERFORMANCE OF THIS SOFTWARE. │ ╚─────────────────────────────────────────────────────────────────────────────*/ -#include "libc/stdio/lock.internal.h" +#include "libc/stdio/lock.h" #include "libc/stdio/stdio.h" /** diff --git a/libc/stdio/fputwc.c b/libc/stdio/fputwc.c index 6356c1aed..44afd3e04 100644 --- a/libc/stdio/fputwc.c +++ b/libc/stdio/fputwc.c @@ -16,7 +16,7 @@ │ TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR │ │ PERFORMANCE OF THIS SOFTWARE. │ ╚─────────────────────────────────────────────────────────────────────────────*/ -#include "libc/stdio/lock.internal.h" +#include "libc/stdio/lock.h" #include "libc/stdio/stdio.h" /** diff --git a/libc/stdio/fputws.c b/libc/stdio/fputws.c index c828e0878..e93151095 100644 --- a/libc/stdio/fputws.c +++ b/libc/stdio/fputws.c @@ -16,7 +16,7 @@ │ TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR │ │ PERFORMANCE OF THIS SOFTWARE. │ ╚─────────────────────────────────────────────────────────────────────────────*/ -#include "libc/stdio/lock.internal.h" +#include "libc/stdio/lock.h" #include "libc/stdio/stdio.h" /** diff --git a/libc/stdio/fread.c b/libc/stdio/fread.c index f5f96007a..f926a9517 100644 --- a/libc/stdio/fread.c +++ b/libc/stdio/fread.c @@ -16,7 +16,7 @@ │ TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR │ │ PERFORMANCE OF THIS SOFTWARE. │ ╚─────────────────────────────────────────────────────────────────────────────*/ -#include "libc/stdio/lock.internal.h" +#include "libc/stdio/lock.h" #include "libc/stdio/stdio.h" /** diff --git a/libc/stdio/freopen.c b/libc/stdio/freopen.c index bcebc8fd7..8799d6775 100644 --- a/libc/stdio/freopen.c +++ b/libc/stdio/freopen.c @@ -17,7 +17,7 @@ │ PERFORMANCE OF THIS SOFTWARE. │ ╚─────────────────────────────────────────────────────────────────────────────*/ #include "libc/calls/calls.h" -#include "libc/stdio/lock.internal.h" +#include "libc/stdio/lock.h" #include "libc/stdio/stdio.h" #include "libc/sysv/consts/f.h" #include "libc/sysv/consts/fd.h" diff --git a/libc/stdio/fseeko.c b/libc/stdio/fseeko.c index d0d8d265f..b29b98c30 100644 --- a/libc/stdio/fseeko.c +++ b/libc/stdio/fseeko.c @@ -16,7 +16,7 @@ │ TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR │ │ PERFORMANCE OF THIS SOFTWARE. │ ╚─────────────────────────────────────────────────────────────────────────────*/ -#include "libc/stdio/lock.internal.h" +#include "libc/stdio/lock.h" #include "libc/stdio/stdio.h" /** diff --git a/libc/stdio/ftello.c b/libc/stdio/ftello.c index 0f12ba119..7ce952ca2 100644 --- a/libc/stdio/ftello.c +++ b/libc/stdio/ftello.c @@ -20,7 +20,7 @@ #include "libc/errno.h" #include "libc/runtime/runtime.h" #include "libc/stdio/internal.h" -#include "libc/stdio/lock.internal.h" +#include "libc/stdio/lock.h" #include "libc/stdio/stdio.h" #include "libc/sysv/consts/o.h" diff --git a/libc/stdio/fwrite.c b/libc/stdio/fwrite.c index 915b0a21c..4522b636b 100644 --- a/libc/stdio/fwrite.c +++ b/libc/stdio/fwrite.c @@ -16,7 +16,7 @@ │ TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR │ │ PERFORMANCE OF THIS SOFTWARE. │ ╚─────────────────────────────────────────────────────────────────────────────*/ -#include "libc/stdio/lock.internal.h" +#include "libc/stdio/lock.h" #include "libc/stdio/stdio.h" /** diff --git a/libc/stdio/getdelim.c b/libc/stdio/getdelim.c index 040cfc968..055c22ead 100644 --- a/libc/stdio/getdelim.c +++ b/libc/stdio/getdelim.c @@ -16,7 +16,7 @@ │ TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR │ │ PERFORMANCE OF THIS SOFTWARE. │ ╚─────────────────────────────────────────────────────────────────────────────*/ -#include "libc/stdio/lock.internal.h" +#include "libc/stdio/lock.h" #include "libc/stdio/stdio.h" /** diff --git a/libc/stdio/iconv.c b/libc/stdio/iconv.c index 8a09fda96..b71ed9160 100644 --- a/libc/stdio/iconv.c +++ b/libc/stdio/iconv.c @@ -27,10 +27,11 @@ ╚─────────────────────────────────────────────────────────────────────────────*/ #include "libc/errno.h" #include "libc/mem/mem.h" +#include "libc/nexgen32e/gettls.h" #include "libc/stdio/iconv.h" -#include "libc/str/locale.h" #include "libc/str/str.h" -#include "libc/thread/tls.h" +#include "libc/thread/thread.h" +#include "libc/str/locale.h" // clang-format off asm(".ident\t\"\\n\\n\ @@ -282,7 +283,7 @@ size_t iconv(iconv_t cd, char **restrict in, size_t *restrict inb, char **restri int err; unsigned char type = map[-1]; unsigned char totype = tomap[-1]; - locale_t *ploc = (locale_t *)&__get_tls()->tib_locale; + locale_t *ploc = &((cthread_t)__get_tls())->locale; locale_t loc = *ploc; if (!in || !*in || !*inb) return 0; diff --git a/libc/stdio/lock.internal.h b/libc/stdio/lock.h similarity index 77% rename from libc/stdio/lock.internal.h rename to libc/stdio/lock.h index 99775cb82..fe38e179d 100644 --- a/libc/stdio/lock.internal.h +++ b/libc/stdio/lock.h @@ -1,11 +1,15 @@ #ifndef COSMOPOLITAN_LIBC_STDIO_LOCK_H_ #define COSMOPOLITAN_LIBC_STDIO_LOCK_H_ -#include "libc/intrin/nopl.internal.h" +#include "libc/intrin/nopl.h" +#include "libc/nexgen32e/threaded.h" #include "libc/stdio/stdio.h" -#include "libc/thread/tls.h" #if !(__ASSEMBLER__ + __LINKER__ + 0) COSMOPOLITAN_C_START_ +void flockfile(FILE *) paramsnonnull(); +void funlockfile(FILE *) paramsnonnull(); +int ftrylockfile(FILE *) paramsnonnull(); + #ifdef _NOPL1 #define flockfile(f) _NOPL1("__threadcalls", flockfile, f) #define funlockfile(f) _NOPL1("__threadcalls", funlockfile, f) diff --git a/libc/stdio/puts.c b/libc/stdio/puts.c index cae291488..aafa06732 100644 --- a/libc/stdio/puts.c +++ b/libc/stdio/puts.c @@ -16,7 +16,7 @@ │ TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR │ │ PERFORMANCE OF THIS SOFTWARE. │ ╚─────────────────────────────────────────────────────────────────────────────*/ -#include "libc/stdio/lock.internal.h" +#include "libc/stdio/lock.h" #include "libc/stdio/stdio.h" #include "libc/str/str.h" diff --git a/libc/stdio/readdir64.S b/libc/stdio/readdir64.S index ea6608a79..471f01bdb 100644 --- a/libc/stdio/readdir64.S +++ b/libc/stdio/readdir64.S @@ -1,5 +1,5 @@ -/*-*- mode:unix-assembly; indent-tabs-mode:t; tab-width:8; coding:utf-8 -*-│ -│vi: set et ft=asm ts=8 tw=8 fenc=utf-8 :vi│ +/*-*- mode:c;indent-tabs-mode:nil;c-basic-offset:2;tab-width:8;coding:utf-8 -*-│ +│vi: set net ft=c ts=2 sts=2 sw=2 fenc=utf-8 :vi│ ╞══════════════════════════════════════════════════════════════════════════════╡ │ This is free and unencumbered software released into the public domain. │ │ │ @@ -26,6 +26,5 @@ ╚─────────────────────────────────────────────────────────────────────────────*/ #include "libc/macros.internal.h" -readdir64: - jmp readdir +readdir64: jmp readdir .endfn readdir64,globl diff --git a/libc/stdio/rewind.c b/libc/stdio/rewind.c index 6ad7a680d..d97050f43 100644 --- a/libc/stdio/rewind.c +++ b/libc/stdio/rewind.c @@ -17,7 +17,7 @@ │ PERFORMANCE OF THIS SOFTWARE. │ ╚─────────────────────────────────────────────────────────────────────────────*/ #include "libc/calls/calls.h" -#include "libc/stdio/lock.internal.h" +#include "libc/stdio/lock.h" #include "libc/stdio/stdio.h" /** diff --git a/libc/stdio/setvbuf.c b/libc/stdio/setvbuf.c index ac58cf08e..1c2aa00eb 100644 --- a/libc/stdio/setvbuf.c +++ b/libc/stdio/setvbuf.c @@ -17,7 +17,7 @@ │ PERFORMANCE OF THIS SOFTWARE. │ ╚─────────────────────────────────────────────────────────────────────────────*/ #include "libc/runtime/runtime.h" -#include "libc/stdio/lock.internal.h" +#include "libc/stdio/lock.h" #include "libc/stdio/stdio.h" #include "libc/sysv/errfuns.h" diff --git a/libc/stdio/stderr-init.S b/libc/stdio/stderr-init.S index 24b3ec928..b5be107f5 100644 --- a/libc/stdio/stderr-init.S +++ b/libc/stdio/stderr-init.S @@ -20,7 +20,7 @@ #include "libc/dce.h" #include "libc/calls/calls.h" #include "libc/sysv/consts/fileno.h" -#include "libc/thread/thread.h" +#include "libc/intrin/pthread.h" #include "libc/macros.internal.h" .init.start 400,_init_stderr @@ -34,6 +34,6 @@ ezlea __stderr_buf,cx mov %rcx,0x18(%rax) #→ f.buf movl $BUFSIZ,0x20(%rax) #→ f.size - movb $PTHREAD_MUTEX_RECURSIVE,0x38+16(%rax) #→ f.lock.attr + movb $PTHREAD_MUTEX_RECURSIVE,0x38(%rax) #→ f.lock.attr mov %rax,stderr(%rip) .init.end 400,_init_stderr,globl,hidden diff --git a/libc/stdio/stdin-init.S b/libc/stdio/stdin-init.S index 86051c810..2f6d03d0d 100644 --- a/libc/stdio/stdin-init.S +++ b/libc/stdio/stdin-init.S @@ -20,7 +20,7 @@ #include "libc/dce.h" #include "libc/calls/calls.h" #include "libc/sysv/consts/fileno.h" -#include "libc/thread/thread.h" +#include "libc/intrin/pthread.h" #include "libc/macros.internal.h" .init.start 400,_init_stdin @@ -30,6 +30,6 @@ ezlea __stdin_buf,cx mov %rcx,0x18(%rax) #→ f.buf movl $BUFSIZ,0x20(%rax) #→ f.size - movb $PTHREAD_MUTEX_RECURSIVE,0x38+16(%rax) #→ f.lock.attr + movb $PTHREAD_MUTEX_RECURSIVE,0x38(%rax) #→ f.lock.attr mov %rax,stdin(%rip) .init.end 400,_init_stdin,globl,hidden diff --git a/libc/stdio/stdio.h b/libc/stdio/stdio.h index 401e344e2..7180a1457 100644 --- a/libc/stdio/stdio.h +++ b/libc/stdio/stdio.h @@ -1,7 +1,7 @@ #ifndef COSMOPOLITAN_LIBC_STDIO_STDIO_H_ #define COSMOPOLITAN_LIBC_STDIO_STDIO_H_ #include "libc/fmt/pflink.h" -#include "libc/thread/thread.h" +#include "libc/intrin/pthread.h" #define _STDIO_H #define L_ctermid 20 @@ -50,9 +50,6 @@ int fileno(FILE *) paramsnonnull() nosideeffect; int fputc(int, FILE *) paramsnonnull(); int fputs(const char *, FILE *) paramsnonnull(); int fputws(const wchar_t *, FILE *) paramsnonnull(); -void flockfile(FILE *) paramsnonnull(); -void funlockfile(FILE *) paramsnonnull(); -int ftrylockfile(FILE *) paramsnonnull(); char *fgets(char *, int, FILE *) paramsnonnull(); wchar_t *fgetws(wchar_t *, int, FILE *) paramsnonnull(); wint_t putwc(wchar_t, FILE *) paramsnonnull(); diff --git a/libc/stdio/stdout-init.S b/libc/stdio/stdout-init.S index 1711b69b6..19c549e10 100644 --- a/libc/stdio/stdout-init.S +++ b/libc/stdio/stdout-init.S @@ -20,7 +20,7 @@ #include "libc/dce.h" #include "libc/calls/calls.h" #include "libc/sysv/consts/fileno.h" -#include "libc/thread/thread.h" +#include "libc/intrin/pthread.h" #include "libc/macros.internal.h" .init.start 400,_init_stdout @@ -32,6 +32,6 @@ ezlea __stdout_buf,cx mov %rcx,0x18(%rax) #→ f.buf movl $BUFSIZ,0x20(%rax) #→ f.size - movb $PTHREAD_MUTEX_RECURSIVE,0x38+16(%rax) #→ f.lock.attr + movb $PTHREAD_MUTEX_RECURSIVE,0x38(%rax) #→ f.lock.attr mov %rax,stdout(%rip) .init.end 400,_init_stdout,globl,hidden diff --git a/libc/stdio/ungetc.c b/libc/stdio/ungetc.c index 7c1bb6535..df066d05e 100644 --- a/libc/stdio/ungetc.c +++ b/libc/stdio/ungetc.c @@ -16,7 +16,7 @@ │ TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR │ │ PERFORMANCE OF THIS SOFTWARE. │ ╚─────────────────────────────────────────────────────────────────────────────*/ -#include "libc/stdio/lock.internal.h" +#include "libc/stdio/lock.h" #include "libc/stdio/stdio.h" /** diff --git a/libc/stdio/ungetwc.c b/libc/stdio/ungetwc.c index 8fb59f6ea..f0c482c9c 100644 --- a/libc/stdio/ungetwc.c +++ b/libc/stdio/ungetwc.c @@ -16,7 +16,7 @@ │ TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR │ │ PERFORMANCE OF THIS SOFTWARE. │ ╚─────────────────────────────────────────────────────────────────────────────*/ -#include "libc/stdio/lock.internal.h" +#include "libc/stdio/lock.h" #include "libc/stdio/stdio.h" /** diff --git a/libc/stdio/vfprintf.c b/libc/stdio/vfprintf.c index 6978a0a26..c44db47ea 100644 --- a/libc/stdio/vfprintf.c +++ b/libc/stdio/vfprintf.c @@ -16,7 +16,7 @@ │ TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR │ │ PERFORMANCE OF THIS SOFTWARE. │ ╚─────────────────────────────────────────────────────────────────────────────*/ -#include "libc/stdio/lock.internal.h" +#include "libc/stdio/lock.h" #include "libc/stdio/stdio.h" /** diff --git a/libc/str/langinfo.c b/libc/str/langinfo.c index 41aad1030..9451b444d 100644 --- a/libc/str/langinfo.c +++ b/libc/str/langinfo.c @@ -25,10 +25,11 @@ │ SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. │ │ │ ╚─────────────────────────────────────────────────────────────────────────────*/ +#include "libc/nexgen32e/gettls.h" +#include "libc/thread/thread.h" #include "libc/str/langinfo.h" #include "libc/str/locale.h" #include "libc/str/nltypes.h" -#include "libc/thread/tls.h" asm(".ident\t\"\\n\\n\ Musl libc (MIT License)\\n\ @@ -100,5 +101,5 @@ char *nl_langinfo_l(nl_item item, locale_t loc) char *nl_langinfo(nl_item item) { - return nl_langinfo_l(item, (locale_t)__get_tls()->tib_locale); + return nl_langinfo_l(item, ((cthread_t)__get_tls())->locale); } diff --git a/libc/stubs/ld.S b/libc/stubs/ld.S index b729f18d2..a3570b917 100644 --- a/libc/stubs/ld.S +++ b/libc/stubs/ld.S @@ -50,8 +50,6 @@ _tdata_size = 0 _tbss_start = 0 _tbss_end = 0 - _tbss_offset = 0 - _tbss_size = 0 _tls_size = 0 _tls_content = 0 @@ -75,8 +73,6 @@ .globl _tdata_size .globl _tbss_start .globl _tbss_end - .globl _tbss_size - .globl _tbss_offset .globl _tls_size .globl _tls_content .globl __data_start @@ -104,10 +100,8 @@ .weak _tdata_size .weak _tbss_start .weak _tbss_end - .weak _tbss_size .weak _tls_size .weak _tls_content - .weak _tbss_offset .weak __data_start .weak __data_end .weak __bss_start diff --git a/libc/sysv/consts.sh b/libc/sysv/consts.sh index 3df60aca6..e548083d7 100755 --- a/libc/sysv/consts.sh +++ b/libc/sysv/consts.sh @@ -579,7 +579,7 @@ syscon sicode SYS_USER_DISPATCH 2 -1 -1 -1 -1 -1 # SIGSYS; syscall # sigaltstack() values # # group name GNU/Systemd XNU's Not UNIX! FreeBSD OpenBSD NetBSD The New Technology Commentary -syscon ss SIGSTKSZ 8192 131072 34816 28672 28672 8192 # overlaid with FRAMESIZE; you need to #undef SIGSTKSZ to access this symbol +syscon ss SIGSTKSZ 8192 131072 34816 28672 28672 8192 # overlaid with STACKSIZE; you need to #undef SIGSTKSZ to access this symbol syscon ss MINSIGSTKSZ 2048 32768 2048 12288 8192 2048 # overlaid with 32768; you need to #undef MINSIGSTKSZ to access this symbol syscon ss SS_ONSTACK 1 1 1 1 1 1 # unix consensus syscon ss SS_DISABLE 2 4 4 4 4 2 # bsd consensus @@ -1316,9 +1316,9 @@ syscon rusage RUSAGE_BOTH -2 99 99 99 99 99 # woop # # group name GNU/Systemd XNU's Not UNIX! FreeBSD OpenBSD NetBSD The New Technology Commentary syscon futex FUTEX_WAIT 0 0 0 1 0 0 -syscon futex FUTEX_WAKE 1 0 0 2 0 0 -syscon futex FUTEX_REQUEUE 3 0 0 3 0 0 -syscon futex FUTEX_PRIVATE_FLAG 128 0 0 128 0 0 +syscon futex FUTEX_WAKE 1 0 0 2 1 0 +syscon futex FUTEX_REQUEUE 3 0 0 3 3 0 +syscon futex FUTEX_PRIVATE_FLAG 128 0 0 128 128 0 # lio_listio() magnums # @@ -1336,7 +1336,7 @@ syscon sched SCHED_OTHER 0 127 2 127 0 127 # standard round-robin syscon sched SCHED_FIFO 1 127 1 127 1 127 # [real-time] first-in, first-out policy syscon sched SCHED_RR 2 127 3 127 2 127 # [real-time] round-robin policy syscon sched SCHED_BATCH 3 127 2 127 0 127 # for "batch" style execution of processes; polyfilled as SCHED_OTHER on non-Linux -syscon sched SCHED_IDLE 5 127 2 127 0 127 # for running very low priority background jobs; polyfilled as SCHED_OTHER on non-Linux +𝔰𝔶𝔰𝔠𝔬𝔫 sched SCHED_IDLE 5 127 2 127 0 127 # for running very low priority background jobs; polyfilled as SCHED_OTHER on non-Linux syscon sched SCHED_DEADLINE 6 127 127 127 127 127 # can only be set by sched_setattr() syscon sched SCHED_RESET_ON_FORK 0x40000000 0 0 0 0 0 # Can be ORed in to make sure the process is reverted back to SCHED_NORMAL on fork(); no-op on non-Linux @@ -1847,7 +1847,7 @@ syscon nr __NR_kill 0x003e 0x2000025 0x0025 0x007a 0x025 0xfff syscon nr __NR_killpg 0xfff 0xfff 0x0092 0xfff 0xfff 0xfff syscon nr __NR_clone 0x0038 0xfff 0xfff 0xfff 0x11f 0xfff syscon nr __NR_tkill 0x00c8 0xfff 0xfff 0xfff 0xfff 0xfff -syscon nr __NR_futex 0x00ca 0xfff 0xfff 0x0053 0xfff 0xfff +syscon nr __NR_futex 0x00ca 0xfff 0xfff 0x0053 0x0a6 0xfff syscon nr __NR_set_robust_list 0x0111 0xfff 0xfff 0xfff 0x0a7 0xfff syscon nr __NR_get_robust_list 0x0112 0xfff 0xfff 0xfff 0x0a8 0xfff syscon nr __NR_uname 0x003f 0xfff 0x00a4 0xfff 0xfff 0xfff diff --git a/libc/sysv/consts/FUTEX_PRIVATE_FLAG.s b/libc/sysv/consts/FUTEX_PRIVATE_FLAG.s index 3d21804a5..da3dd6143 100644 --- a/libc/sysv/consts/FUTEX_PRIVATE_FLAG.s +++ b/libc/sysv/consts/FUTEX_PRIVATE_FLAG.s @@ -1,2 +1,2 @@ .include "o/libc/sysv/consts/syscon.internal.inc" -.syscon futex,FUTEX_PRIVATE_FLAG,128,0,0,128,0,0 +.syscon futex,FUTEX_PRIVATE_FLAG,128,0,0,128,128,0 diff --git a/libc/sysv/consts/FUTEX_REQUEUE.s b/libc/sysv/consts/FUTEX_REQUEUE.s index e25e5dffb..b0a8baa21 100644 --- a/libc/sysv/consts/FUTEX_REQUEUE.s +++ b/libc/sysv/consts/FUTEX_REQUEUE.s @@ -1,2 +1,2 @@ .include "o/libc/sysv/consts/syscon.internal.inc" -.syscon futex,FUTEX_REQUEUE,3,0,0,3,0,0 +.syscon futex,FUTEX_REQUEUE,3,0,0,3,3,0 diff --git a/libc/sysv/consts/FUTEX_WAKE.s b/libc/sysv/consts/FUTEX_WAKE.s index acdc80afa..2d6184318 100644 --- a/libc/sysv/consts/FUTEX_WAKE.s +++ b/libc/sysv/consts/FUTEX_WAKE.s @@ -1,2 +1,2 @@ .include "o/libc/sysv/consts/syscon.internal.inc" -.syscon futex,FUTEX_WAKE,1,0,0,2,0,0 +.syscon futex,FUTEX_WAKE,1,0,0,2,1,0 diff --git a/libc/sysv/consts/SOMAXCONN.s b/libc/sysv/consts/SOMAXCONN.s index f711831ce..5c6a6ef77 100644 --- a/libc/sysv/consts/SOMAXCONN.s +++ b/libc/sysv/consts/SOMAXCONN.s @@ -1,2 +1,2 @@ .include "o/libc/sysv/consts/syscon.internal.inc" -.syscon limits,SOMAXCONN,4096,128,128,128,128,2147483647 +.syscon misc,SOMAXCONN,0x80,0x80,0x80,0x80,0x80,0x7fffffff diff --git a/libc/sysv/consts/__NR_futex.s b/libc/sysv/consts/__NR_futex.s index a7bd77d79..bbcc0b744 100644 --- a/libc/sysv/consts/__NR_futex.s +++ b/libc/sysv/consts/__NR_futex.s @@ -1,2 +1,2 @@ .include "o/libc/sysv/consts/syscon.internal.inc" -.syscon nr,__NR_futex,0x00ca,0xfff,0xfff,0x0053,0xfff,0xfff +.syscon nr,__NR_futex,0x00ca,0xfff,0xfff,0x0053,0x0a6,0xfff diff --git a/libc/sysv/consts/futex.h b/libc/sysv/consts/futex.h index 91641a3d7..f6b12cf2b 100644 --- a/libc/sysv/consts/futex.h +++ b/libc/sysv/consts/futex.h @@ -2,25 +2,21 @@ #define COSMOPOLITAN_LIBC_SYSV_CONSTS_FUTEX_H_ #include "libc/runtime/symbolic.h" -#define FUTEX_WAIT SYMBOLIC(FUTEX_WAIT) -#define FUTEX_WAKE SYMBOLIC(FUTEX_WAKE) -#define FUTEX_REQUEUE SYMBOLIC(FUTEX_REQUEUE) -#define FUTEX_PRIVATE_FLAG 128 - +#define FUTEX_WAIT SYMBOLIC(FUTEX_WAIT) +#define FUTEX_WAKE SYMBOLIC(FUTEX_WAKE) +#define FUTEX_REQUEUE SYMBOLIC(FUTEX_REQUEUE) +#define FUTEX_PRIVATE_FLAG SYMBOLIC(FUTEX_PRIVATE_FLAG) #define FUTEX_WAIT_PRIVATE (FUTEX_WAIT | FUTEX_PRIVATE_FLAG) #define FUTEX_WAKE_PRIVATE (FUTEX_WAKE | FUTEX_PRIVATE_FLAG) #define FUTEX_REQUEUE_PRIVATE (FUTEX_REQUEUE | FUTEX_PRIVATE_FLAG) -#define FUTEX_WAIT_BITSET 9 -#define FUTEX_CLOCK_REALTIME 256 -#define FUTEX_BITSET_MATCH_ANY 0xffffffff - #if !(__ASSEMBLER__ + __LINKER__ + 0) COSMOPOLITAN_C_START_ extern const int FUTEX_WAIT; extern const int FUTEX_WAKE; extern const int FUTEX_REQUEUE; +extern const int FUTEX_PRIVATE_FLAG; COSMOPOLITAN_C_END_ #endif /* !(__ASSEMBLER__ + __LINKER__ + 0) */ diff --git a/libc/sysv/consts/ss.h b/libc/sysv/consts/ss.h index 9f05e231c..68d135fae 100644 --- a/libc/sysv/consts/ss.h +++ b/libc/sysv/consts/ss.h @@ -10,7 +10,7 @@ extern const int SS_DISABLE; COSMOPOLITAN_C_END_ #endif /* !(__ASSEMBLER__ + __LINKER__ + 0) */ -#define SIGSTKSZ FRAMESIZE +#define SIGSTKSZ STACKSIZE #define MINSIGSTKSZ 32768 #define SS_ONSTACK 1 #define SS_DISABLE SS_DISABLE diff --git a/libc/sysv/errno_location.greg.c b/libc/sysv/errno_location.greg.c index fe1fc4cd6..425939f73 100644 --- a/libc/sysv/errno_location.greg.c +++ b/libc/sysv/errno_location.greg.c @@ -17,12 +17,16 @@ │ PERFORMANCE OF THIS SOFTWARE. │ ╚─────────────────────────────────────────────────────────────────────────────*/ #include "libc/errno.h" -#include "libc/thread/tls2.h" +#include "libc/nexgen32e/gettls.h" +#include "libc/nexgen32e/threaded.h" /** * Returns address of errno variable. + * + * @see __initialize_tls() + * @see __install_tls() */ privileged nocallersavedregisters errno_t *(__errno_location)(void) { if (!__tls_enabled) return &__errno; - return &__get_tls_privileged()->tib_errno; + return (errno_t *)(__get_tls_privileged() + 0x3c); } diff --git a/libc/sysv/strace.greg.c b/libc/sysv/strace.greg.c index 302fc69c1..0822d65cb 100644 --- a/libc/sysv/strace.greg.c +++ b/libc/sysv/strace.greg.c @@ -16,7 +16,6 @@ │ TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR │ │ PERFORMANCE OF THIS SOFTWARE. │ ╚─────────────────────────────────────────────────────────────────────────────*/ -#include "libc/atomic.h" #include "libc/runtime/runtime.h" /** @@ -41,4 +40,4 @@ * under normal circumstances, `__strace` should only be either zero or * one. */ -atomic_int __strace; +_Atomic(int) __strace; diff --git a/libc/sysv/syscall.S b/libc/sysv/syscall.S index 6cffdcf8a..275a656da 100644 --- a/libc/sysv/syscall.S +++ b/libc/sysv/syscall.S @@ -16,7 +16,6 @@ │ TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR │ │ PERFORMANCE OF THIS SOFTWARE. │ ╚─────────────────────────────────────────────────────────────────────────────*/ -#include "libc/dce.h" #include "libc/macros.internal.h" .privileged @@ -33,52 +32,12 @@ // @param %rsi,%rdx,%rcx,%r8,%r9 may supply parameters 1 through 5 // @param sixth is optionally pushed on the stack before call // @return %rax has result, or -1 w/ errno on failure -syscall: - push %rbp - mov %rsp,%rbp - .profilable - -// slide arguments into their right places - mov %rdi,%rax # nr - mov %rsi,%rdi # arg 1 - mov %rdx,%rsi # arg 2 - mov %rcx,%rdx # arg 3 - mov %r8,%rcx # arg 4 - mov %r9,%r8 # arg 5 - mov 16(%rbp),%r9 # arg 6 - push 32(%rbp) # arg 8 - push 24(%rbp) # arg 7 - -// convert from consts.sh to syscalls.sh encoding - push %rcx - mov __hostos(%rip),%cl - test $LINUX,%cl - jnz 2f -1: test $FREEBSD,%cl - jz 1f - shl $4*7,%rax - jmp 2f -1: test $OPENBSD,%cl - jz 1f - shl $4*10,%rax - jmp 2f -1: test $NETBSD,%cl - jz 1f - shl $4*13,%rax - jmp 2f -1: test $XNU,%cl - jz 2f - mov %eax,%ecx - and $0x0f000000,%ecx - and $0x00000fff,%eax - shl $4*3,%eax - or %ecx,%eax -2: pop %rcx - -// trigger the system call - call *__systemfive(%rip) - -// clean up stack and return - leave - ret +syscall:mov %rdi,%rax + mov %rsi,%rdi + mov %rdx,%rsi + mov %rcx,%rdx + mov %r8,%rcx # ← intended + mov %r9,%r8 + mov 8(%rsp),%r9 + jmp *__systemfive(%rip) .endfn syscall,globl diff --git a/libc/testlib/globals.c b/libc/testlib/globals.c index e844ce117..c6563c30d 100644 --- a/libc/testlib/globals.c +++ b/libc/testlib/globals.c @@ -16,9 +16,8 @@ │ TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR │ │ PERFORMANCE OF THIS SOFTWARE. │ ╚─────────────────────────────────────────────────────────────────────────────*/ -#include "libc/atomic.h" #include "libc/testlib/testlib.h" char g_fixturename[256]; -atomic_uint g_testlib_ran; -atomic_uint g_testlib_failed; +_Atomic(unsigned) g_testlib_ran; +_Atomic(unsigned) g_testlib_failed; diff --git a/libc/testlib/testmain.c b/libc/testlib/testmain.c index c15144785..af8ed85fa 100644 --- a/libc/testlib/testmain.c +++ b/libc/testlib/testmain.c @@ -159,8 +159,6 @@ static void SetLimit(int resource, uint64_t soft, uint64_t hard) { } } -#pragma weak main - /** * Generic test program main function. */ @@ -184,13 +182,13 @@ noasan int main(int argc, char *argv[]) { testlib_runalltests(); if (!g_testlib_failed && runbenchmarks_ && weaken(testlib_runallbenchmarks)) { weaken(testlib_runallbenchmarks)(); - if (IsAsan() && !g_testlib_failed) { + if (!g_testlib_failed) { CheckForMemoryLeaks(); } if (!g_testlib_failed && IsRunningUnderMake()) { return 254; // compile.com considers this 0 and propagates output } - } else if (IsAsan() && !g_testlib_failed) { + } else if (!g_testlib_failed) { CheckForMemoryLeaks(); } diff --git a/libc/testlib/testrunner.c b/libc/testlib/testrunner.c index ea8627a26..bd67be048 100644 --- a/libc/testlib/testrunner.c +++ b/libc/testlib/testrunner.c @@ -28,7 +28,7 @@ #include "libc/fmt/fmt.h" #include "libc/fmt/itoa.h" #include "libc/intrin/atomic.h" -#include "libc/thread/thread.h" +#include "libc/intrin/pthread.h" #include "libc/intrin/weaken.h" #include "libc/log/check.h" #include "libc/log/internal.h" diff --git a/libc/thread/README.md b/libc/thread/README.md deleted file mode 100644 index 99c6a61d4..000000000 --- a/libc/thread/README.md +++ /dev/null @@ -1,5 +0,0 @@ -# Cosmpolitan POSIX Threads Library - -Cosmopolitan Libc implements threading as it is written in The Open -Group Base Specifications Issue 7, 2018 edition IEEE Std 1003.1-2017 -(Revision of IEEE Std 1003.1-2008) in addition to GNU extensions. diff --git a/libc/thread/internal.h b/libc/thread/internal.h new file mode 100644 index 000000000..331e7899b --- /dev/null +++ b/libc/thread/internal.h @@ -0,0 +1,15 @@ +#ifndef COSMOPOLITAN_LIBC_THREAD_INTERNAL_H_ +#define COSMOPOLITAN_LIBC_THREAD_INTERNAL_H_ +#include "libc/intrin/pthread.h" +#if !(__ASSEMBLER__ + __LINKER__ + 0) +COSMOPOLITAN_C_START_ + +hidden extern uint64_t _pthread_key_usage[(PTHREAD_KEYS_MAX + 63) / 64]; +hidden extern pthread_key_dtor _pthread_key_dtor[PTHREAD_KEYS_MAX]; +hidden extern _Thread_local void *_pthread_keys[PTHREAD_KEYS_MAX]; + +void _pthread_key_destruct(void *[PTHREAD_KEYS_MAX]); + +COSMOPOLITAN_C_END_ +#endif /* !(__ASSEMBLER__ + __LINKER__ + 0) */ +#endif /* COSMOPOLITAN_LIBC_THREAD_INTERNAL_H_ */ diff --git a/libc/intrin/pthread_keys.c b/libc/thread/key.c similarity index 94% rename from libc/intrin/pthread_keys.c rename to libc/thread/key.c index 69be8bafd..63d917840 100644 --- a/libc/intrin/pthread_keys.c +++ b/libc/thread/key.c @@ -16,10 +16,7 @@ │ TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR │ │ PERFORMANCE OF THIS SOFTWARE. │ ╚─────────────────────────────────────────────────────────────────────────────*/ -#include "libc/thread/posixthread.internal.h" -#include "libc/thread/thread.h" - -pthread_spinlock_t _pthread_keys_lock; +#include "libc/thread/internal.h" // tls value slots for pthread keys api _Thread_local void *_pthread_keys[PTHREAD_KEYS_MAX]; diff --git a/libc/thread/mktls.c b/libc/thread/mktls.c index 8eccb9014..911c2a18e 100644 --- a/libc/thread/mktls.c +++ b/libc/thread/mktls.c @@ -16,20 +16,18 @@ │ TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR │ │ PERFORMANCE OF THIS SOFTWARE. │ ╚─────────────────────────────────────────────────────────────────────────────*/ -#include "libc/dce.h" -#include "libc/intrin/asan.internal.h" -#include "libc/intrin/asancodes.h" #include "libc/macros.internal.h" #include "libc/mem/mem.h" #include "libc/runtime/internal.h" #include "libc/runtime/runtime.h" #include "libc/str/str.h" #include "libc/thread/spawn.h" -#include "libc/thread/tls.h" +#include "libc/thread/thread.h" -#define I(x) ((intptr_t)x) - -void Bzero(void *, size_t) asm("bzero"); // gcc bug +#define _TLSZ ((intptr_t)_tls_size) +#define _TLDZ ((intptr_t)_tdata_size) +#define _TIBZ sizeof(struct cthread_descriptor_t) +#define _MEMZ ROUNDUP(_TLSZ + _TIBZ, _Alignof(struct cthread_descriptor_t)) /** * Allocates thread-local storage memory for new thread. @@ -37,27 +35,19 @@ void Bzero(void *, size_t) asm("bzero"); // gcc bug */ char *_mktls(char **out_tib) { char *tls; - struct CosmoTib *tib; + cthread_t tib; - // allocate memory for tdata, tbss, and tib - tls = memalign(TLS_ALIGNMENT, I(_tls_size) + sizeof(struct CosmoTib)); - if (!tls) return 0; - - // poison memory between tdata and tbss - if (IsAsan()) { - __asan_poison(tls + I(_tdata_size), I(_tbss_offset) - I(_tdata_size), - kAsanProtected); - } - - // initialize tdata and clear tbss - memmove(tls, _tdata_start, I(_tdata_size)); - Bzero(tls + I(_tbss_offset), I(_tbss_size) + sizeof(struct CosmoTib)); + // Allocate enough TLS memory for all the GNU Linuker (_tls_size) + // organized _Thread_local data, as well as Cosmpolitan Libc (64) + if (!(tls = calloc(1, _MEMZ))) return 0; // set up thread information block - tib = (struct CosmoTib *)(tls + I(_tls_size)); - tib->tib_self = tib; - tib->tib_self2 = tib; - tib->tib_tid = -1; + tib = (cthread_t)(tls + _MEMZ - _TIBZ); + tib->self = tib; + tib->self2 = tib; + tib->err = 0; + tib->tid = -1; + memmove(tls, _tdata_start, _TLDZ); if (out_tib) { *out_tib = (char *)tib; diff --git a/libc/thread/nsync_counter.c b/libc/thread/nsync_counter.c deleted file mode 100644 index dddc67ec4..000000000 --- a/libc/thread/nsync_counter.c +++ /dev/null @@ -1,158 +0,0 @@ -/*-*- mode:c;indent-tabs-mode:t;c-basic-offset:8;tab-width:8;coding:utf-8 -*-│ -│vi: set et ft=c ts=8 tw=8 fenc=utf-8 :vi│ -╞══════════════════════════════════════════════════════════════════════════════╡ -│ Copyright 2016 Google Inc. │ -│ │ -│ Licensed under the Apache License, Version 2.0 (the "License"); │ -│ you may not use this file except in compliance with the License. │ -│ You may obtain a copy of the License at │ -│ │ -│ http://www.apache.org/licenses/LICENSE-2.0 │ -│ │ -│ Unless required by applicable law or agreed to in writing, software │ -│ distributed under the License is distributed on an "AS IS" BASIS, │ -│ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. │ -│ See the License for the specific language governing permissions and │ -│ limitations under the License. │ -╚─────────────────────────────────────────────────────────────────────────────*/ -#include "libc/mem/mem.h" -#include "libc/str/str.h" -#include "third_party/nsync/atomic.h" -#include "third_party/nsync/atomic.internal.h" -#include "third_party/nsync/common.internal.h" -#include "third_party/nsync/counter.h" -#include "third_party/nsync/dll.h" -#include "third_party/nsync/mu_semaphore.h" -#include "third_party/nsync/races.internal.h" -#include "third_party/nsync/wait_s.internal.h" -#include "third_party/nsync/waiter.h" - -asm(".ident\t\"\\n\\n\ -*NSYNC (Apache 2.0)\\n\ -Copyright 2016 Google, Inc.\\n\ -https://github.com/google/nsync\""); -// clang-format off - -/* Internal details of nsync_counter. */ -struct nsync_counter_s_ { - nsync_atomic_uint32_ waited; /* wait has been called */ - nsync_mu counter_mu; /* protects fields below except reads of "value" */ - nsync_atomic_uint32_ value; /* value of counter */ - struct nsync_dll_element_s_ *waiters; /* list of waiters */ -}; - -nsync_counter nsync_counter_new (uint32_t value) { - nsync_counter c = (nsync_counter) malloc (sizeof (*c)); - if (c != NULL) { - memset ((void *) c, 0, sizeof (*c)); - ATM_STORE (&c->value, value); - } - return (c); -} - -void nsync_counter_free (nsync_counter c) { - nsync_mu_lock (&c->counter_mu); - ASSERT (nsync_dll_is_empty_ (c->waiters)); - nsync_mu_unlock (&c->counter_mu); - free (c); -} - -uint32_t nsync_counter_add (nsync_counter c, int32_t delta) { - uint32_t value; - IGNORE_RACES_START (); - if (delta == 0) { - value = ATM_LOAD_ACQ (&c->value); - } else { - nsync_mu_lock (&c->counter_mu); - do { - value = ATM_LOAD (&c->value); - } while (!ATM_CAS_RELACQ (&c->value, value, value+delta)); - value += delta; - if (delta > 0) { - /* It's illegal to increase the count from zero if - there has been a waiter. */ - ASSERT (value != (uint32_t) delta || !ATM_LOAD (&c->waited)); - ASSERT (value > value - delta); /* Crash on overflow. */ - } else { - ASSERT (value < value - delta); /* Crash on overflow. */ - } - if (value == 0) { - nsync_dll_element_ *p; - while ((p = nsync_dll_first_ (c->waiters)) != NULL) { - struct nsync_waiter_s *nw = DLL_NSYNC_WAITER (p); - c->waiters = nsync_dll_remove_ (c->waiters, p); - ATM_STORE_REL (&nw->waiting, 0); - nsync_mu_semaphore_v (nw->sem); - } - } - nsync_mu_unlock (&c->counter_mu); - } - IGNORE_RACES_END (); - return (value); -} - -uint32_t nsync_counter_value (nsync_counter c) { - uint32_t result; - IGNORE_RACES_START (); - result = ATM_LOAD_ACQ (&c->value); - IGNORE_RACES_END (); - return (result); -} - -uint32_t nsync_counter_wait (nsync_counter c, nsync_time abs_deadline) { - struct nsync_waitable_s waitable; - struct nsync_waitable_s *pwaitable = &waitable; - uint32_t result = 0; - waitable.v = c; - waitable.funcs = &nsync_counter_waitable_funcs; - if (nsync_wait_n (NULL, NULL, NULL, abs_deadline, 1, &pwaitable) != 0) { - IGNORE_RACES_START (); - result = ATM_LOAD_ACQ (&c->value); - IGNORE_RACES_END (); - } - return (result); -} - -static nsync_time counter_ready_time (void *v, struct nsync_waiter_s *nw) { - nsync_counter c = (nsync_counter) v; - nsync_time r; - ATM_STORE (&c->waited, 1); - r = (ATM_LOAD_ACQ (&c->value) == 0? nsync_time_zero : nsync_time_no_deadline); - return (r); -} - -static int counter_enqueue (void *v, struct nsync_waiter_s *nw) { - nsync_counter c = (nsync_counter) v; - int32_t value; - nsync_mu_lock (&c->counter_mu); - value = ATM_LOAD_ACQ (&c->value); - if (value != 0) { - c->waiters = nsync_dll_make_last_in_list_ (c->waiters, &nw->q); - ATM_STORE (&nw->waiting, 1); - } else { - ATM_STORE (&nw->waiting, 0); - } - nsync_mu_unlock (&c->counter_mu); - return (value != 0); -} - -static int counter_dequeue (void *v, struct nsync_waiter_s *nw) { - nsync_counter c = (nsync_counter) v; - int32_t value; - nsync_mu_lock (&c->counter_mu); - value = ATM_LOAD_ACQ (&c->value); - if (ATM_LOAD_ACQ (&nw->waiting) != 0) { - c->waiters = nsync_dll_remove_ (c->waiters, &nw->q); - ATM_STORE (&nw->waiting, 0); - } - nsync_mu_unlock (&c->counter_mu); - return (value != 0); -} - -const struct nsync_waitable_funcs_s nsync_counter_waitable_funcs = { - &counter_ready_time, - &counter_enqueue, - &counter_dequeue -}; - - diff --git a/libc/thread/nsync_cv.c b/libc/thread/nsync_cv.c deleted file mode 100644 index 5b859f939..000000000 --- a/libc/thread/nsync_cv.c +++ /dev/null @@ -1,497 +0,0 @@ -/*-*- mode:c;indent-tabs-mode:t;c-basic-offset:8;tab-width:8;coding:utf-8 -*-│ -│vi: set et ft=c ts=8 tw=8 fenc=utf-8 :vi│ -╞══════════════════════════════════════════════════════════════════════════════╡ -│ Copyright 2016 Google Inc. │ -│ │ -│ Licensed under the Apache License, Version 2.0 (the "License"); │ -│ you may not use this file except in compliance with the License. │ -│ You may obtain a copy of the License at │ -│ │ -│ http://www.apache.org/licenses/LICENSE-2.0 │ -│ │ -│ Unless required by applicable law or agreed to in writing, software │ -│ distributed under the License is distributed on an "AS IS" BASIS, │ -│ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. │ -│ See the License for the specific language governing permissions and │ -│ limitations under the License. │ -╚─────────────────────────────────────────────────────────────────────────────*/ -#include "libc/str/str.h" -#include "third_party/nsync/atomic.internal.h" -#include "third_party/nsync/common.internal.h" -#include "third_party/nsync/cv.h" -#include "third_party/nsync/dll.h" -#include "third_party/nsync/races.internal.h" -#include "third_party/nsync/wait_s.internal.h" -#include "third_party/nsync/waiter.h" - -asm(".ident\t\"\\n\\n\ -*NSYNC (Apache 2.0)\\n\ -Copyright 2016 Google, Inc.\\n\ -https://github.com/google/nsync\""); -// clang-format off - -/* Initialize *cv. */ -void nsync_cv_init (nsync_cv *cv) { - memset ((void *) cv, 0, sizeof (*cv)); -} - -/* Wake the cv waiters in the circular list pointed to by - to_wake_list, which may not be NULL. If the waiter is associated with a - nsync_mu, the "wakeup" may consist of transferring the waiters to the nsync_mu's - queue. Requires that every waiter is associated with the same mutex. - all_readers indicates whether all the waiters on the list are readers. */ -static void wake_waiters (nsync_dll_list_ to_wake_list, int all_readers) { - nsync_dll_element_ *p = NULL; - nsync_dll_element_ *next = NULL; - nsync_dll_element_ *first_waiter = nsync_dll_first_ (to_wake_list); - struct nsync_waiter_s *first_nw = DLL_NSYNC_WAITER (first_waiter); - waiter *first_w = NULL; - nsync_mu *pmu = NULL; - if ((first_nw->flags & NSYNC_WAITER_FLAG_MUCV) != 0) { - first_w = DLL_WAITER (first_waiter); - pmu = first_w->cv_mu; - } - if (pmu != NULL) { /* waiter is associated with the nsync_mu *pmu. */ - /* We will transfer elements of to_wake_list to *pmu if all of: - - some thread holds the lock, and - - *pmu's spinlock is not held, and - - either *pmu cannot be acquired in the mode of the first - waiter, or there's more than one thread on to_wake_list - and not all are readers, and - - we acquire the spinlock on the first try. - The spinlock acquisition also marks *pmu as having waiters. - The requirement that some thread holds the lock ensures - that at least one of the transferred waiters will be woken. - */ - uint32_t old_mu_word = ATM_LOAD (&pmu->word); - int first_cant_acquire = ((old_mu_word & first_w->l_type->zero_to_acquire) != 0); - next = nsync_dll_next_ (to_wake_list, first_waiter); - if ((old_mu_word&MU_ANY_LOCK) != 0 && - (old_mu_word&MU_SPINLOCK) == 0 && - (first_cant_acquire || (next != NULL && !all_readers)) && - ATM_CAS_ACQ (&pmu->word, old_mu_word, - (old_mu_word|MU_SPINLOCK|MU_WAITING) & - ~MU_ALL_FALSE)) { - - uint32_t set_on_release = 0; - - /* For any waiter that should be transferred, rather - than woken, move it from to_wake_list to pmu->waiters. */ - int first_is_writer = first_w->l_type == nsync_writer_type_; - int transferred_a_writer = 0; - int woke_areader = 0; - /* Transfer the first waiter iff it can't acquire *pmu. */ - if (first_cant_acquire) { - to_wake_list = nsync_dll_remove_ (to_wake_list, first_waiter); - pmu->waiters = nsync_dll_make_last_in_list_ (pmu->waiters, first_waiter); - /* tell nsync_cv_wait_with_deadline() that we - moved the waiter to *pmu's queue. */ - first_w->cv_mu = NULL; - /* first_nw.waiting is already 1, from being on - cv's waiter queue. */ - transferred_a_writer = first_is_writer; - } else { - woke_areader = !first_is_writer; - } - /* Now process the other waiters. */ - for (p = next; p != NULL; p = next) { - int p_is_writer; - struct nsync_waiter_s *p_nw = DLL_NSYNC_WAITER (p); - waiter *p_w = NULL; - if ((p_nw->flags & NSYNC_WAITER_FLAG_MUCV) != 0) { - p_w = DLL_WAITER (p); - } - next = nsync_dll_next_ (to_wake_list, p); - p_is_writer = (p_w != NULL && - DLL_WAITER (p)->l_type == nsync_writer_type_); - /* We transfer this element if any of: - - the first waiter can't acquire *pmu, or - - the first waiter is a writer, or - - this element is a writer. */ - if (p_w == NULL) { - /* wake non-native waiter */ - } else if (first_cant_acquire || first_is_writer || p_is_writer) { - to_wake_list = nsync_dll_remove_ (to_wake_list, p); - pmu->waiters = nsync_dll_make_last_in_list_ (pmu->waiters, p); - /* tell nsync_cv_wait_with_deadline() - that we moved the waiter to *pmu's - queue. */ - p_w->cv_mu = NULL; - /* p_nw->waiting is already 1, from - being on cv's waiter queue. */ - transferred_a_writer = transferred_a_writer || p_is_writer; - } else { - woke_areader = woke_areader || !p_is_writer; - } - } - - /* Claim a waiting writer if we transferred one, except if we woke readers, - in which case we want those readers to be able to acquire immediately. */ - if (transferred_a_writer && !woke_areader) { - set_on_release |= MU_WRITER_WAITING; - } - - /* release *pmu's spinlock (MU_WAITING was set by CAS above) */ - old_mu_word = ATM_LOAD (&pmu->word); - while (!ATM_CAS_REL (&pmu->word, old_mu_word, - (old_mu_word|set_on_release) & ~MU_SPINLOCK)) { - old_mu_word = ATM_LOAD (&pmu->word); - } - } - } - - /* Wake any waiters we didn't manage to enqueue on the mu. */ - for (p = nsync_dll_first_ (to_wake_list); p != NULL; p = next) { - struct nsync_waiter_s *p_nw = DLL_NSYNC_WAITER (p); - next = nsync_dll_next_ (to_wake_list, p); - to_wake_list = nsync_dll_remove_ (to_wake_list, p); - /* Wake the waiter. */ - ATM_STORE_REL (&p_nw->waiting, 0); /* release store */ - nsync_mu_semaphore_v (p_nw->sem); - } -} - -/* ------------------------------------------ */ - -/* Versions of nsync_mu_lock() and nsync_mu_unlock() that take "void *" - arguments, to avoid call through a function pointer of a different type, - which is undefined. */ -static void void_mu_lock (void *mu) { - nsync_mu_lock ((nsync_mu *) mu); -} -static void void_mu_unlock (void *mu) { - nsync_mu_unlock ((nsync_mu *) mu); -} - -/* Atomically release *pmu (which must be held on entry) - and block the calling thread on *pcv. Then wait until awakened by a - call to nsync_cv_signal() or nsync_cv_broadcast() (or a spurious wakeup), or by the time - reaching abs_deadline, or by cancel_note being notified. In all cases, - reacquire *pmu, and return the reason for the call returned (0, ETIMEDOUT, - or ECANCELED). Callers should abs_deadline==nsync_time_no_deadline for no - deadline, and cancel_note==NULL for no cancellation. nsync_cv_wait_with_deadline() - should be used in a loop, as with all Mesa-style condition variables. See - examples above. - - There are two reasons for using an absolute deadline, rather than a relative - timeout---these are why pthread_cond_timedwait() also uses an absolute - deadline. First, condition variable waits have to be used in a loop; with - an absolute times, the deadline does not have to be recomputed on each - iteration. Second, in most real programmes, some activity (such as an RPC - to a server, or when guaranteeing response time in a UI), there is a - deadline imposed by the specification or the caller/user; relative delays - can shift arbitrarily with scheduling delays, and so after multiple waits - might extend beyond the expected deadline. Relative delays tend to be more - convenient mostly in tests and trivial examples than they are in real - programmes. */ -int nsync_cv_wait_with_deadline_generic (nsync_cv *pcv, void *pmu, - void (*lock) (void *), void (*unlock) (void *), - nsync_time abs_deadline, - nsync_note cancel_note) { - nsync_mu *cv_mu = NULL; - int is_reader_mu; - uint32_t old_word; - uint32_t remove_count; - int sem_outcome; - unsigned attempts; - int outcome = 0; - waiter *w; - IGNORE_RACES_START (); - w = nsync_waiter_new_ (); - ATM_STORE (&w->nw.waiting, 1); - w->cond.f = NULL; /* Not using a conditional critical section. */ - w->cond.v = NULL; - w->cond.eq = NULL; - if (lock == &void_mu_lock || - lock == (void (*) (void *)) &nsync_mu_lock || - lock == (void (*) (void *)) &nsync_mu_rlock) { - cv_mu = (nsync_mu *) pmu; - } - w->cv_mu = cv_mu; /* If *pmu is an nsync_mu, record its address, else record NULL. */ - is_reader_mu = 0; /* If true, an nsync_mu in reader mode. */ - if (cv_mu == NULL) { - w->l_type = NULL; - } else { - uint32_t old_mu_word = ATM_LOAD (&cv_mu->word); - int is_writer = (old_mu_word & MU_WHELD_IF_NON_ZERO) != 0; - int is_reader = (old_mu_word & MU_RHELD_IF_NON_ZERO) != 0; - if (is_writer) { - if (is_reader) { - nsync_panic_ ("mu held in reader and writer mode simultaneously " - "on entry to nsync_cv_wait_with_deadline()\n"); - } - w->l_type = nsync_writer_type_; - } else if (is_reader) { - w->l_type = nsync_reader_type_; - is_reader_mu = 1; - } else { - nsync_panic_ ("mu not held on entry to nsync_cv_wait_with_deadline()\n"); - } - } - - /* acquire spinlock, set non-empty */ - old_word = nsync_spin_test_and_set_ (&pcv->word, CV_SPINLOCK, CV_SPINLOCK|CV_NON_EMPTY, 0); - pcv->waiters = nsync_dll_make_last_in_list_ (pcv->waiters, &w->nw.q); - remove_count = ATM_LOAD (&w->remove_count); - /* Release the spin lock. */ - ATM_STORE_REL (&pcv->word, old_word|CV_NON_EMPTY); /* release store */ - - /* Release *pmu. */ - if (is_reader_mu) { - nsync_mu_runlock (cv_mu); - } else { - (*unlock) (pmu); - } - - /* wait until awoken or a timeout. */ - sem_outcome = 0; - attempts = 0; - while (ATM_LOAD_ACQ (&w->nw.waiting) != 0) { /* acquire load */ - if (sem_outcome == 0) { - sem_outcome = nsync_sem_wait_with_cancel_ (w, abs_deadline, cancel_note); - } - - if (sem_outcome != 0 && ATM_LOAD (&w->nw.waiting) != 0) { - /* A timeout or cancellation occurred, and no wakeup. - Acquire *pcv's spinlock, and confirm. */ - old_word = nsync_spin_test_and_set_ (&pcv->word, CV_SPINLOCK, - CV_SPINLOCK, 0); - /* Check that w wasn't removed from the queue after we - checked above, but before we acquired the spinlock. - The test of remove_count confirms that the waiter *w - is still governed by *pcv's spinlock; otherwise, some - other thread is about to set w.waiting==0. */ - if (ATM_LOAD (&w->nw.waiting) != 0) { - if (remove_count == ATM_LOAD (&w->remove_count)) { - uint32_t old_value; - /* still in cv waiter queue */ - /* Not woken, so remove *w from cv - queue, and declare a - timeout/cancellation. */ - outcome = sem_outcome; - pcv->waiters = nsync_dll_remove_ (pcv->waiters, - &w->nw.q); - do { - old_value = ATM_LOAD (&w->remove_count); - } while (!ATM_CAS (&w->remove_count, old_value, old_value+1)); - if (nsync_dll_is_empty_ (pcv->waiters)) { - old_word &= ~(CV_NON_EMPTY); - } - ATM_STORE_REL (&w->nw.waiting, 0); /* release store */ - } - } - /* Release spinlock. */ - ATM_STORE_REL (&pcv->word, old_word); /* release store */ - } - - if (ATM_LOAD (&w->nw.waiting) != 0) { - /* The delay here causes this thread ultimately to - yield to another that has dequeued this thread, but - has not yet set the waiting field to zero; a - cancellation or timeout may prevent this thread - from blocking above on the semaphore. */ - attempts = nsync_spin_delay_ (attempts); - } - } - - if (cv_mu != NULL && w->cv_mu == NULL) { /* waiter was moved to *pmu's queue, and woken. */ - /* Requeue on *pmu using existing waiter struct; current thread - is the designated waker. */ - nsync_mu_lock_slow_ (cv_mu, w, MU_DESIG_WAKER, w->l_type); - nsync_waiter_free_ (w); - } else { - /* Traditional case: We've woken from the cv, and need to reacquire *pmu. */ - nsync_waiter_free_ (w); - if (is_reader_mu) { - nsync_mu_rlock (cv_mu); - } else { - (*lock) (pmu); - } - } - IGNORE_RACES_END (); - return (outcome); -} - -/* Wake at least one thread if any are currently blocked on *pcv. If - the chosen thread is a reader on an nsync_mu, wake all readers and, if - possible, a writer. */ -void nsync_cv_signal (nsync_cv *pcv) { - IGNORE_RACES_START (); - if ((ATM_LOAD_ACQ (&pcv->word) & CV_NON_EMPTY) != 0) { /* acquire load */ - nsync_dll_list_ to_wake_list = NULL; /* waiters that we will wake */ - int all_readers = 0; - /* acquire spinlock */ - uint32_t old_word = nsync_spin_test_and_set_ (&pcv->word, CV_SPINLOCK, - CV_SPINLOCK, 0); - if (!nsync_dll_is_empty_ (pcv->waiters)) { - /* Point to first waiter that enqueued itself, and - detach it from all others. */ - struct nsync_waiter_s *first_nw; - nsync_dll_element_ *first = nsync_dll_first_ (pcv->waiters); - pcv->waiters = nsync_dll_remove_ (pcv->waiters, first); - first_nw = DLL_NSYNC_WAITER (first); - if ((first_nw->flags & NSYNC_WAITER_FLAG_MUCV) != 0) { - uint32_t old_value; - do { - old_value = - ATM_LOAD (&DLL_WAITER (first)->remove_count); - } while (!ATM_CAS (&DLL_WAITER (first)->remove_count, - old_value, old_value+1)); - } - to_wake_list = nsync_dll_make_last_in_list_ (to_wake_list, first); - if ((first_nw->flags & NSYNC_WAITER_FLAG_MUCV) != 0 && - DLL_WAITER (first)->l_type == nsync_reader_type_) { - int woke_writer; - /* If the first waiter is a reader, wake all readers, and - if it's possible, one writer. This allows reader-regions - to be added to a monitor without invalidating code in which - a client has optimized broadcast calls by converting them to - signal calls. In particular, we wake a writer when waking - readers because the readers will not invalidate the condition - that motivated the client to call nsync_cv_signal(). But we - wake at most one writer because the first writer may invalidate - the condition; the client is expecting only one writer to be - able make use of the wakeup, or he would have called - nsync_cv_broadcast(). */ - nsync_dll_element_ *p = NULL; - nsync_dll_element_ *next = NULL; - all_readers = 1; - woke_writer = 0; - for (p = nsync_dll_first_ (pcv->waiters); p != NULL; p = next) { - struct nsync_waiter_s *p_nw = DLL_NSYNC_WAITER (p); - int should_wake; - next = nsync_dll_next_ (pcv->waiters, p); - should_wake = 0; - if ((p_nw->flags & NSYNC_WAITER_FLAG_MUCV) != 0 && - DLL_WAITER (p)->l_type == nsync_reader_type_) { - should_wake = 1; - } else if (!woke_writer) { - woke_writer = 1; - all_readers = 0; - should_wake = 1; - } - if (should_wake) { - pcv->waiters = nsync_dll_remove_ (pcv->waiters, p); - if ((p_nw->flags & NSYNC_WAITER_FLAG_MUCV) != 0) { - uint32_t old_value; - do { - old_value = ATM_LOAD ( - &DLL_WAITER (p)->remove_count); - } while (!ATM_CAS (&DLL_WAITER (p)->remove_count, - old_value, old_value+1)); - } - to_wake_list = nsync_dll_make_last_in_list_ ( - to_wake_list, p); - } - } - } - if (nsync_dll_is_empty_ (pcv->waiters)) { - old_word &= ~(CV_NON_EMPTY); - } - } - /* Release spinlock. */ - ATM_STORE_REL (&pcv->word, old_word); /* release store */ - if (!nsync_dll_is_empty_ (to_wake_list)) { - wake_waiters (to_wake_list, all_readers); - } - } - IGNORE_RACES_END (); -} - -/* Wake all threads currently blocked on *pcv. */ -void nsync_cv_broadcast (nsync_cv *pcv) { - IGNORE_RACES_START (); - if ((ATM_LOAD_ACQ (&pcv->word) & CV_NON_EMPTY) != 0) { /* acquire load */ - nsync_dll_element_ *p; - nsync_dll_element_ *next; - int all_readers; - nsync_dll_list_ to_wake_list = NULL; /* waiters that we will wake */ - /* acquire spinlock */ - nsync_spin_test_and_set_ (&pcv->word, CV_SPINLOCK, CV_SPINLOCK, 0); - p = NULL; - next = NULL; - all_readers = 1; - /* Wake entire waiter list, which we leave empty. */ - for (p = nsync_dll_first_ (pcv->waiters); p != NULL; p = next) { - struct nsync_waiter_s *p_nw = DLL_NSYNC_WAITER (p); - next = nsync_dll_next_ (pcv->waiters, p); - all_readers = all_readers && (p_nw->flags & NSYNC_WAITER_FLAG_MUCV) != 0 && - (DLL_WAITER (p)->l_type == nsync_reader_type_); - pcv->waiters = nsync_dll_remove_ (pcv->waiters, p); - if ((p_nw->flags & NSYNC_WAITER_FLAG_MUCV) != 0) { - uint32_t old_value; - do { - old_value = ATM_LOAD (&DLL_WAITER (p)->remove_count); - } while (!ATM_CAS (&DLL_WAITER (p)->remove_count, - old_value, old_value+1)); - } - to_wake_list = nsync_dll_make_last_in_list_ (to_wake_list, p); - } - /* Release spinlock and mark queue empty. */ - ATM_STORE_REL (&pcv->word, 0); /* release store */ - if (!nsync_dll_is_empty_ (to_wake_list)) { /* Wake them. */ - wake_waiters (to_wake_list, all_readers); - } - } - IGNORE_RACES_END (); -} - -/* Wait with deadline, using an nsync_mu. */ -int nsync_cv_wait_with_deadline (nsync_cv *pcv, nsync_mu *pmu, - nsync_time abs_deadline, - nsync_note cancel_note) { - return (nsync_cv_wait_with_deadline_generic (pcv, pmu, &void_mu_lock, - &void_mu_unlock, - abs_deadline, cancel_note)); -} - -/* Atomically release *pmu and block the caller on *pcv. Wait - until awakened by a call to nsync_cv_signal() or nsync_cv_broadcast(), or a spurious - wakeup. Then reacquires *pmu, and return. The call is equivalent to a call - to nsync_cv_wait_with_deadline() with abs_deadline==nsync_time_no_deadline, and a NULL - cancel_note. It should be used in a loop, as with all standard Mesa-style - condition variables. See examples above. */ -void nsync_cv_wait (nsync_cv *pcv, nsync_mu *pmu) { - nsync_cv_wait_with_deadline (pcv, pmu, nsync_time_no_deadline, NULL); -} - -static nsync_time cv_ready_time (void *v, struct nsync_waiter_s *nw) { - nsync_time r; - r = (nw == NULL || ATM_LOAD_ACQ (&nw->waiting) != 0? nsync_time_no_deadline : nsync_time_zero); - return (r); -} - -static int cv_enqueue (void *v, struct nsync_waiter_s *nw) { - nsync_cv *pcv = (nsync_cv *) v; - /* acquire spinlock */ - uint32_t old_word = nsync_spin_test_and_set_ (&pcv->word, CV_SPINLOCK, CV_SPINLOCK, 0); - pcv->waiters = nsync_dll_make_last_in_list_ (pcv->waiters, &nw->q); - ATM_STORE (&nw->waiting, 1); - /* Release spinlock. */ - ATM_STORE_REL (&pcv->word, old_word | CV_NON_EMPTY); /* release store */ - return (1); -} - -static int cv_dequeue (void *v, struct nsync_waiter_s *nw) { - nsync_cv *pcv = (nsync_cv *) v; - int was_queued = 0; - /* acquire spinlock */ - uint32_t old_word = nsync_spin_test_and_set_ (&pcv->word, CV_SPINLOCK, CV_SPINLOCK, 0); - if (ATM_LOAD_ACQ (&nw->waiting) != 0) { - pcv->waiters = nsync_dll_remove_ (pcv->waiters, &nw->q); - ATM_STORE (&nw->waiting, 0); - was_queued = 1; - } - if (nsync_dll_is_empty_ (pcv->waiters)) { - old_word &= ~(CV_NON_EMPTY); - } - /* Release spinlock. */ - ATM_STORE_REL (&pcv->word, old_word); /* release store */ - return (was_queued); -} - -const struct nsync_waitable_funcs_s nsync_cv_waitable_funcs = { - &cv_ready_time, - &cv_enqueue, - &cv_dequeue -}; diff --git a/libc/thread/nsync_debug.c b/libc/thread/nsync_debug.c deleted file mode 100644 index abfeb73ba..000000000 --- a/libc/thread/nsync_debug.c +++ /dev/null @@ -1,292 +0,0 @@ -/*-*- mode:c;indent-tabs-mode:t;c-basic-offset:8;tab-width:8;coding:utf-8 -*-│ -│vi: set et ft=c ts=8 tw=8 fenc=utf-8 :vi│ -╞══════════════════════════════════════════════════════════════════════════════╡ -│ Copyright 2016 Google Inc. │ -│ │ -│ Licensed under the Apache License, Version 2.0 (the "License"); │ -│ you may not use this file except in compliance with the License. │ -│ You may obtain a copy of the License at │ -│ │ -│ http://www.apache.org/licenses/LICENSE-2.0 │ -│ │ -│ Unless required by applicable law or agreed to in writing, software │ -│ distributed under the License is distributed on an "AS IS" BASIS, │ -│ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. │ -│ See the License for the specific language governing permissions and │ -│ limitations under the License. │ -╚─────────────────────────────────────────────────────────────────────────────*/ -#include "third_party/nsync/atomic.h" -#include "third_party/nsync/common.internal.h" -#include "third_party/nsync/dll.h" -#include "third_party/nsync/mu_semaphore.h" -#include "third_party/nsync/races.internal.h" -#include "third_party/nsync/wait_s.internal.h" - -asm(".ident\t\"\\n\\n\ -*NSYNC (Apache 2.0)\\n\ -Copyright 2016 Google, Inc.\\n\ -https://github.com/google/nsync\""); -// clang-format off - -/* Routines for debugging. */ - -/* An emit_buf represents a buffer into which debug information can - be written. */ -struct emit_buf { - char *start; /* start of buffer */ - int len; /* pength of buffer */ - int pos; /* position of next character to bve written */ - int overflow; /* non-zero iff buffer overflow has occurred */ -}; - -/* Initialize *b to point to start[0, .., len-1], and return b. - of to an internal static buffer if buf==NULL. */ -static struct emit_buf *emit_init (struct emit_buf *b, char *start, int len) { - b->start = start; - b->len = len; - b->pos = 0; - b->overflow = 0; - return (b); -} - - -/* Write character c to buffer *b. */ -static void emit_c (struct emit_buf *b, int c) { - if (b->pos < b->len) { - b->start[b->pos++] = c; - } else if (!b->overflow) { - static const char suffix[] = "..."; - const char *s = &suffix[sizeof (suffix)]; /* past nul */ - char *p = &b->start[b->len]; /* past end */ - while (s > suffix && p > b->start) { - *--p = *--s; - } - b->overflow = 1; - } -} - -/* A printf-like function that writes to an emit_buf. - It understands only the format specifiers %s (const char *), and %i - (uintptr_t in hex), with no modifiers. */ -static void emit_print (struct emit_buf *b, const char *fmt, ...) { - va_list ap; - va_start (ap, fmt); - while (*fmt != 0) { - int c = *fmt++; - if (c != '%') { - emit_c (b, c); - } else { - c = *fmt++; - if (c == 's') { - const char *s = va_arg (ap, const char *); - while (*s != 0) { - emit_c (b, *s++); - } - } else if (c == 'i') { - uintptr_t n = va_arg (ap, uintptr_t); - int i; - for (i = 0; (n >> i) >= 0x10; i += 4) { - } - for (; i >= 0; i -= 4) { - emit_c (b, "0123456789abcdef"[(n >> i) & 0xf]); - } - } else { - ASSERT (0); - } - } - } - va_end (ap); -} - -/* Map a bit in a uint32_t to a human-readable name. */ -struct bit_name { - uint32_t mask; - const char *name; -}; - -/* names for bits in a mu word */ -static const struct bit_name mu_bit[] = { - { MU_WLOCK, "wlock" }, - { MU_SPINLOCK, "spin" }, - { MU_WAITING, "wait" }, - { MU_DESIG_WAKER, "desig" }, - { MU_CONDITION, "cond" }, - { MU_WRITER_WAITING, "writer" }, - { MU_LONG_WAIT, "long" }, - { MU_ALL_FALSE, "false" }, - { 0, "" } /* sentinel */ -}; - -/* names for bits in a cv word */ -static const struct bit_name cv_bit[] = { - { CV_SPINLOCK, "spin" }, - { CV_NON_EMPTY, "wait" }, - { 0, "" } /* sentinel */ -}; - -/* names for bits in a waiter flags word */ -static const struct bit_name waiter_flags_bit[] = { - { WAITER_RESERVED, "rsrvd" }, - { WAITER_IN_USE, "in_use" }, - { 0, "" } /* sentinel */ -}; - -/* Emit the names of bits in word to buffer *b using names[] */ -static void emit_word (struct emit_buf *b, const struct bit_name *name, uint32_t word) { - int i; - for (i = 0; name[i].mask != 0; i++) { - if ((word & name[i].mask) != 0) { - emit_print (b, " %s", name[i].name); - } - } -} - -/* Emit the waiter queue *q to *b. */ -static void emit_waiters (struct emit_buf *b, nsync_dll_list_ list) { - nsync_dll_element_ *p = nsync_dll_first_ (list); - nsync_dll_element_ *next; - if (p != NULL) { - emit_print (b, "\nwaiters =\n"); - } - for (; p != NULL && !b->overflow; p = next) { - struct nsync_waiter_s *nw = DLL_NSYNC_WAITER (p); - waiter *w = DLL_WAITER (p); - next = NULL; - emit_print (b, " %i", (uintptr_t) w); - if (w->tag != WAITER_TAG) { - emit_print (b, "bad WAITER_TAG %i", - (uintptr_t) w->tag); - } else { - next = nsync_dll_next_ (list, p); - if (nw->tag != NSYNC_WAITER_TAG) { - emit_print (b, " bad WAITER_TAG %i", - (uintptr_t) nw->tag); - } else { - emit_print (b, " embedded=%i waiting=%i", - (uintptr_t) (w->flags & NSYNC_WAITER_FLAG_MUCV), - (uintptr_t) ATM_LOAD (&nw->waiting)); - } - emit_word (b, waiter_flags_bit, w->flags); - emit_print (b, " %s removes=%i cond=(%i %i %i)", - w->l_type == nsync_writer_type_? "writer" : - w->l_type == nsync_reader_type_? "reader" : - "??????", - (uintptr_t) ATM_LOAD (&w->remove_count), - (uintptr_t) w->cond.f, - (uintptr_t) w->cond.v, - (uintptr_t) w->cond.eq); - if (w->same_condition.next != &w->same_condition) { - emit_print (b, " same_as %i", - (uintptr_t) DLL_WAITER_SAMECOND ( - w->same_condition.next)); - } - } - emit_c (b, '\n'); - } -} - -/* Emit to *b the state of *mu, and return a pointer to *b's buffer. - - If blocking!=0, print_waiters!=0, and *mu's waiter list is non-empty, the - call will block until it can acquire the spinlock. - If print_waiters!=0, the waiter list is printed. - The spinlock is released before return if it was acquired. - blocking==0 && print_waiters!=0 is unsafe and is intended for use within - interactive debuggers. */ -static char *emit_mu_state (struct emit_buf *b, nsync_mu *mu, - int blocking, int print_waiters) { - uintptr_t word; - uintptr_t readers; - int acquired = 0; - IGNORE_RACES_START (); - word = ATM_LOAD (&mu->word); - if ((word & MU_WAITING) != 0 && print_waiters && /* can benefit from lock */ - (blocking || (word & MU_SPINLOCK) == 0)) { /* willing, or no need to wait */ - word = nsync_spin_test_and_set_ (&mu->word, MU_SPINLOCK, MU_SPINLOCK, 0); - acquired = 1; - } - readers = word / MU_RLOCK; - emit_print (b, "mu 0x%i -> 0x%i = {", (uintptr_t) mu, word); - emit_word (b, mu_bit, word); - if (readers != 0) { - emit_print (b, " readers=0x%i", readers); - } - emit_print (b, " }"); - if (print_waiters) { - emit_waiters (b, mu->waiters); - } - if (acquired) { - ATM_STORE_REL (&mu->word, word); /* release store */ - } - emit_c (b, 0); - IGNORE_RACES_END (); - return (b->start); -} - -/* Emit to *b the state of *cv, and return a pointer to *b's buffer. - - If blocking!=0, print_waiters!=0, and *cv's waiter list is non-empty, the - call will block until it can acquire the spinlock. - If print_waiters!=0, the waiter list is printed. - The spinlock is released before return if it was acquired. - blocking==0 && print_waiters!=0 is unsafe and is intended for use within - interactive debuggers. */ -static char *emit_cv_state (struct emit_buf *b, nsync_cv *cv, - int blocking, int print_waiters) { - uintptr_t word; - int acquired = 0; - IGNORE_RACES_START (); - word = ATM_LOAD (&cv->word); - if ((word & CV_NON_EMPTY) != 0 && print_waiters && /* can benefit from lock */ - (blocking || (word & CV_SPINLOCK) == 0)) { /* willing, or no need to wait */ - word = nsync_spin_test_and_set_ (&cv->word, CV_SPINLOCK, CV_SPINLOCK, 0); - acquired = 1; - } - emit_print (b, "cv 0x%i -> 0x%i = {", (uintptr_t) cv, word); - emit_word (b, cv_bit, word); - emit_print (b, " }"); - if (print_waiters) { - emit_waiters (b, cv->waiters); - } - if (acquired) { - ATM_STORE_REL (&cv->word, word); /* release store */ - } - emit_c (b, 0); - IGNORE_RACES_END (); - return (b->start); -} - -char *nsync_mu_debug_state (nsync_mu *mu, char *buf, int n) { - struct emit_buf b; - return (emit_mu_state (emit_init (&b, buf, n), mu, 0, 0)); -} - -char *nsync_cv_debug_state (nsync_cv *cv, char *buf, int n) { - struct emit_buf b; - return (emit_cv_state (emit_init (&b, buf, n), cv, 0, 0)); -} - -char *nsync_mu_debug_state_and_waiters (nsync_mu *mu, char *buf, int n) { - struct emit_buf b; - return (emit_mu_state (emit_init (&b, buf, n), mu, 1, 1)); -} - -char *nsync_cv_debug_state_and_waiters (nsync_cv *cv, char *buf, int n) { - struct emit_buf b; - return (emit_cv_state (emit_init (&b, buf, n), cv, 1, 1)); -} - -static char nsync_debug_buf[1024]; - -char *nsync_mu_debugger (nsync_mu *mu) { - struct emit_buf b; - return (emit_mu_state (emit_init (&b, nsync_debug_buf, - (int) sizeof (nsync_debug_buf)), - mu, 0, 1)); -} -char *nsync_cv_debugger (nsync_cv *cv) { - struct emit_buf b; - return (emit_cv_state (emit_init (&b, nsync_debug_buf, - (int) sizeof (nsync_debug_buf)), - cv, 0, 1)); -} diff --git a/libc/thread/nsync_mu_wait.c b/libc/thread/nsync_mu_wait.c deleted file mode 100644 index 9b409e239..000000000 --- a/libc/thread/nsync_mu_wait.c +++ /dev/null @@ -1,322 +0,0 @@ -/*-*- mode:c;indent-tabs-mode:t;c-basic-offset:8;tab-width:8;coding:utf-8 -*-│ -│vi: set et ft=c ts=8 tw=8 fenc=utf-8 :vi│ -╞══════════════════════════════════════════════════════════════════════════════╡ -│ Copyright 2016 Google Inc. │ -│ │ -│ Licensed under the Apache License, Version 2.0 (the "License"); │ -│ you may not use this file except in compliance with the License. │ -│ You may obtain a copy of the License at │ -│ │ -│ http://www.apache.org/licenses/LICENSE-2.0 │ -│ │ -│ Unless required by applicable law or agreed to in writing, software │ -│ distributed under the License is distributed on an "AS IS" BASIS, │ -│ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. │ -│ See the License for the specific language governing permissions and │ -│ limitations under the License. │ -╚─────────────────────────────────────────────────────────────────────────────*/ -#include "third_party/nsync/atomic.h" -#include "third_party/nsync/common.internal.h" -#include "third_party/nsync/dll.h" -#include "third_party/nsync/mu_semaphore.h" -#include "third_party/nsync/races.internal.h" -#include "third_party/nsync/wait_s.internal.h" - -asm(".ident\t\"\\n\\n\ -*NSYNC (Apache 2.0)\\n\ -Copyright 2016 Google, Inc.\\n\ -https://github.com/google/nsync\""); -// clang-format off - -/* Attempt to remove waiter *w from *mu's - waiter queue. If successful, leave the lock held in mode *l_type, and - return non-zero; otherwise return zero. Requires that the current thread - hold neither *mu nor its spinlock, that remove_count be the value of - w.remove_count when *w was inserted into the queue (which it will still be if - it has not been removed). - - This is a tricky part of the design. Here is the rationale. - - When a condition times out or is cancelled, we must "turn off" the - condition, making it always true, so the lock will be acquired in the normal - way. The naive approach would be to set a field atomically to tell future - waiters to ignore the condition. Unfortunately, that would violate the - same_condition list invariants, and the same_condition optimization is - probably worth keeping. - - To fixup the same_condition list, we must have mutual exclusion with the loop - in nsync_mu_unlock_slow_() that is examining waiters, evaluating their conditions, and - removing them from the queue. That loop uses both the spinlock (to allow - queue changes), and the mutex itself (to allow condition evaluation). - Therefore, to "turn off" the condition requires acquiring both the spinlock - and the mutex. This has two consequences: - - If we must acquire *mu to "turn off" the condition, we might as well give - the lock to this waiter and return from nsync_cv_wait_with_deadline() after we've - done so. It would be wasted work to put it back on the waiter queue, and - have it wake up and acquire yet again. (There are possibilities for - starvation here that we ignore, under the assumption that the client - avoids timeouts that are extremely short relative to the durations of his - section durations.) - - We can't use *w to wait for the lock to be free, because *w is already on - the waiter queue with the wrong condition; we now want to wait with no - condition. So either we must spin to acquire the lock, or we must - allocate _another_ waiter object. The latter option is feasible, but - delicate: the thread would have two waiter objects, and would have to - handle being woken by either one or both, and possibly removing one that - was not awoken. For the moment, we spin, because it's easier, and seems - not to cause problems in practice, since the spinloop backs off - aggressively. */ -static int mu_try_acquire_after_timeout_or_cancel (nsync_mu *mu, lock_type *l_type, - waiter *w, uint32_t remove_count) { - int success = 0; - unsigned spin_attempts = 0; - uint32_t old_word = ATM_LOAD (&mu->word); - /* Spin until we can acquire the spinlock and a writer lock on *mu. */ - while ((old_word&(MU_WZERO_TO_ACQUIRE|MU_SPINLOCK)) != 0 || - !ATM_CAS_ACQ (&mu->word, old_word, - (old_word+MU_WADD_TO_ACQUIRE+MU_SPINLOCK) & - ~MU_WCLEAR_ON_ACQUIRE)) { - /* Failed to acquire. If we can, set the MU_WRITER_WAITING bit - to avoid being starved by readers. */ - if ((old_word & (MU_WRITER_WAITING | MU_SPINLOCK)) == 0) { - /* If the following CAS succeeds, it effectively - acquires and releases the spinlock atomically, so - must be both an acquire and release barrier. - MU_WRITER_WAITING will be cleared via - MU_WCLEAR_ON_ACQUIRE when this loop succeeds. - An optimization; failures are ignored. */ - ATM_CAS_RELACQ (&mu->word, old_word, - old_word|MU_WRITER_WAITING); - } - spin_attempts = nsync_spin_delay_ (spin_attempts); - old_word = ATM_LOAD (&mu->word); - } - /* Check that w wasn't removed from the queue after our caller checked, - but before we acquired the spinlock. - The check of remove_count confirms that the waiter *w is still - governed by *mu's spinlock. Otherwise, some other thread may be - about to set w.waiting==0. */ - if (ATM_LOAD (&w->nw.waiting) != 0 && remove_count == ATM_LOAD (&w->remove_count)) { - /* This thread's condition is now irrelevant, and it - holds a writer lock. Remove it from the queue, - and possibly convert back to a reader lock. */ - mu->waiters = nsync_remove_from_mu_queue_ (mu->waiters, &w->nw.q); - ATM_STORE (&w->nw.waiting, 0); - - /* Release spinlock but keep desired lock type. */ - ATM_STORE_REL (&mu->word, old_word+l_type->add_to_acquire); /* release store */ - success = 1; - } else { - /* Release spinlock and *mu. */ - ATM_STORE_REL (&mu->word, old_word); /* release store */ - } - return (success); -} - -/* Return when at least one of: the condition is true, the - deadline expires, or cancel_note is notified. It may unlock and relock *mu - while blocked waiting for one of these events, but always returns with *mu - held. It returns 0 iff the condition is true on return, and otherwise - either ETIMEDOUT or ECANCELED, depending on why the call returned early. Use - abs_deadline==nsync_time_no_deadline for no deadline, and cancel_note==NULL for no - cancellation. - - Requires that *mu be held on entry. - Requires that condition.eval() neither modify state protected by *mu, nor - return a value dependent on state not protected by *mu. To depend on time, - use the abs_deadline parameter. - (Conventional use of condition variables have the same restrictions on the - conditions tested by the while-loop.) - The implementation calls condition.eval() only with *mu held, though not - always from the calling thread, and may elect to hold only a read lock - during the call, even if the client is attempting to acquire only write - locks. - - The nsync_mu_wait() and nsync_mu_wait_with_deadline() calls can be used instead of condition - variables. In many straightforward situations they are of equivalent - performance and are somewhat easier to use, because unlike condition - variables, they do not require that the waits be placed in a loop, and they - do not require explicit wakeup calls. In the current implementation, use of - nsync_mu_wait() and nsync_mu_wait_with_deadline() can take longer if many distinct - wait conditions are used. In such cases, use an explicit condition variable - per wakeup condition for best performance. */ -int nsync_mu_wait_with_deadline (nsync_mu *mu, - int (*condition) (const void *condition_arg), - const void *condition_arg, - int (*condition_arg_eq) (const void *a, const void *b), - nsync_time abs_deadline, nsync_note cancel_note) { - lock_type *l_type; - int first_wait; - int condition_is_true; - waiter *w; - int outcome; - /* Work out in which mode the lock is held. */ - uint32_t old_word; - IGNORE_RACES_START (); - old_word = ATM_LOAD (&mu->word); - if ((old_word & MU_ANY_LOCK) == 0) { - nsync_panic_ ("nsync_mu not held in some mode when calling " - "nsync_mu_wait_with_deadline()\n"); - } - l_type = nsync_writer_type_; - if ((old_word & MU_RHELD_IF_NON_ZERO) != 0) { - l_type = nsync_reader_type_; - } - - first_wait = 1; /* first time through the loop below. */ - condition_is_true = (condition == NULL || (*condition) (condition_arg)); - - /* Loop until either the condition becomes true, or "outcome" indicates - cancellation or timeout. */ - w = NULL; - outcome = 0; - while (outcome == 0 && !condition_is_true) { - uint32_t has_condition; - uint32_t remove_count; - uint32_t add_to_acquire; - int had_waiters; - int sem_outcome; - unsigned attempts; - int have_lock; - if (w == NULL) { - w = nsync_waiter_new_ (); /* get a waiter struct if we need one. */ - } - - /* Prepare to wait. */ - w->cv_mu = NULL; /* not a condition variable wait */ - w->l_type = l_type; - w->cond.f = condition; - w->cond.v = condition_arg; - w->cond.eq = condition_arg_eq; - has_condition = 0; /* set to MU_CONDITION if condition is non-NULL */ - if (condition != NULL) { - has_condition = MU_CONDITION; - } - ATM_STORE (&w->nw.waiting, 1); - remove_count = ATM_LOAD (&w->remove_count); - - /* Acquire spinlock. */ - old_word = nsync_spin_test_and_set_ (&mu->word, MU_SPINLOCK, - MU_SPINLOCK|MU_WAITING|has_condition, MU_ALL_FALSE); - had_waiters = ((old_word & (MU_DESIG_WAKER | MU_WAITING)) == MU_WAITING); - /* Queue the waiter. */ - if (first_wait) { - nsync_maybe_merge_conditions_ (nsync_dll_last_ (mu->waiters), - &w->nw.q); - /* first wait goes to end of queue */ - mu->waiters = nsync_dll_make_last_in_list_ (mu->waiters, - &w->nw.q); - first_wait = 0; - } else { - nsync_maybe_merge_conditions_ (&w->nw.q, - nsync_dll_first_ (mu->waiters)); - /* subsequent waits go to front of queue */ - mu->waiters = nsync_dll_make_first_in_list_ (mu->waiters, - &w->nw.q); - } - /* Release spinlock and *mu. */ - do { - old_word = ATM_LOAD (&mu->word); - add_to_acquire = l_type->add_to_acquire; - if (((old_word-l_type->add_to_acquire)&MU_ANY_LOCK) == 0 && had_waiters) { - add_to_acquire = 0; /* release happens in nsync_mu_unlock_slow_ */ - } - } while (!ATM_CAS_REL (&mu->word, old_word, - (old_word - add_to_acquire) & ~MU_SPINLOCK)); - if (add_to_acquire == 0) { - /* The lock will be fully released, there are waiters, and - no designated waker, so wake waiters. */ - nsync_mu_unlock_slow_ (mu, l_type); - } - - /* wait until awoken or a timeout. */ - sem_outcome = 0; - attempts = 0; - have_lock = 0; - while (ATM_LOAD_ACQ (&w->nw.waiting) != 0) { /* acquire load */ - if (sem_outcome == 0) { - sem_outcome = nsync_sem_wait_with_cancel_ (w, abs_deadline, - cancel_note); - if (sem_outcome != 0 && ATM_LOAD (&w->nw.waiting) != 0) { - /* A timeout or cancellation occurred, and no wakeup. - Acquire the spinlock and mu, and confirm. */ - have_lock = mu_try_acquire_after_timeout_or_cancel ( - mu, l_type, w, remove_count); - if (have_lock) { /* Successful acquire. */ - outcome = sem_outcome; - } - } - } - - if (ATM_LOAD (&w->nw.waiting) != 0) { - attempts = nsync_spin_delay_ (attempts); /* will ultimately yield */ - } - } - - if (!have_lock) { - /* If we didn't reacquire due to a cancellation/timeout, acquire now. */ - nsync_mu_lock_slow_ (mu, w, MU_DESIG_WAKER, l_type); - } - condition_is_true = (condition == NULL || (*condition) (condition_arg)); - } - if (w != NULL) { - nsync_waiter_free_ (w); /* free waiter if we allocated one. */ - } - if (condition_is_true) { - outcome = 0; /* condition is true trumps other outcomes. */ - } - IGNORE_RACES_END (); - return (outcome); -} - -/* Return when the condition is true. Perhaps unlock and relock *mu - while blocked waiting for the condition to become true. It is equivalent to - a call to nsync_mu_wait_with_deadline() with abs_deadline==nsync_time_no_deadline, and - cancel_note==NULL. - - Requires that *mu be held on entry. - Calls condition.eval() only with *mu held, though not always from the - calling thread. - See wait_with_deadline() for the restrictions on condition and performance - considerations. */ -void nsync_mu_wait (nsync_mu *mu, int (*condition) (const void *condition_arg), - const void *condition_arg, - int (*condition_arg_eq) (const void *a, const void *b)) { - if (nsync_mu_wait_with_deadline (mu, condition, condition_arg, condition_arg_eq, - nsync_time_no_deadline, NULL) != 0) { - nsync_panic_ ("nsync_mu_wait woke but condition not true\n"); - } -} - -/* Unlock *mu, which must be held in write mode, and wake waiters, if - appropriate. Unlike nsync_mu_unlock(), this call is not required to wake - nsync_mu_wait/nsync_mu_wait_with_deadline calls on conditions that were - false before this thread acquired the lock. This call should be used only - at the end of critical sections for which: - - nsync_mu_wait/nsync_mu_wait_with_deadline are in use on the same mutex, - - this critical section cannot make the condition true for any of those - nsync_mu_wait/nsync_mu_wait_with_deadline waits, and - - when performance is significantly improved by doing so. */ -void nsync_mu_unlock_without_wakeup (nsync_mu *mu) { - IGNORE_RACES_START (); - /* See comment in nsync_mu_unlock(). */ - if (!ATM_CAS_REL (&mu->word, MU_WLOCK, 0)) { - uint32_t old_word = ATM_LOAD (&mu->word); - uint32_t new_word = old_word - MU_WLOCK; - if ((new_word & (MU_RLOCK_FIELD | MU_WLOCK)) != 0) { - if ((old_word & MU_RLOCK_FIELD) != 0) { - nsync_panic_ ("attempt to nsync_mu_unlock() an nsync_mu " - "held in read mode\n"); - } else { - nsync_panic_ ("attempt to nsync_mu_unlock() an nsync_mu " - "not held in write mode\n"); - } - } else if ((old_word & (MU_WAITING | MU_DESIG_WAKER | MU_ALL_FALSE)) == - MU_WAITING || !ATM_CAS_REL (&mu->word, old_word, new_word)) { - nsync_mu_unlock_slow_ (mu, nsync_writer_type_); - } - } - IGNORE_RACES_END (); -} - - diff --git a/libc/thread/nsync_note.c b/libc/thread/nsync_note.c deleted file mode 100644 index f13b0a582..000000000 --- a/libc/thread/nsync_note.c +++ /dev/null @@ -1,307 +0,0 @@ -/*-*- mode:c;indent-tabs-mode:t;c-basic-offset:8;tab-width:8;coding:utf-8 -*-│ -│vi: set et ft=c ts=8 tw=8 fenc=utf-8 :vi│ -╞══════════════════════════════════════════════════════════════════════════════╡ -│ Copyright 2016 Google Inc. │ -│ │ -│ Licensed under the Apache License, Version 2.0 (the "License"); │ -│ you may not use this file except in compliance with the License. │ -│ You may obtain a copy of the License at │ -│ │ -│ http://www.apache.org/licenses/LICENSE-2.0 │ -│ │ -│ Unless required by applicable law or agreed to in writing, software │ -│ distributed under the License is distributed on an "AS IS" BASIS, │ -│ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. │ -│ See the License for the specific language governing permissions and │ -│ limitations under the License. │ -╚─────────────────────────────────────────────────────────────────────────────*/ -#include "libc/mem/mem.h" -#include "libc/str/str.h" -#include "third_party/nsync/atomic.h" -#include "third_party/nsync/common.internal.h" -#include "third_party/nsync/dll.h" -#include "third_party/nsync/mu_semaphore.h" -#include "third_party/nsync/mu_wait.h" -#include "third_party/nsync/races.internal.h" -#include "third_party/nsync/wait_s.internal.h" -#include "third_party/nsync/waiter.h" - -asm(".ident\t\"\\n\\n\ -*NSYNC (Apache 2.0)\\n\ -Copyright 2016 Google, Inc.\\n\ -https://github.com/google/nsync\""); -// clang-format off - -/* Locking discipline for the nsync_note implementation: - - Each nsync_note has a lock "note_mu" which protects the "parent" pointer, - "waiters" list, and "disconnecting" count. It also protects the "children" - list; thus each node's "parent_child_link", which links together the - children of a single parent, is protected by the parent's "note_mu". - - To connect a parent to a child, or to disconnect one, the parent's lock must - be held to manipulate its child list, and the child's lock must be held to - change the parent pointer, so both must be held simultaneously. - The locking order is "parent before child". - - Operations like notify and free are given a node pointer n and must - disconnect *n from its parent n->parent. The call must hold n->note_mu to - read n->parent, but need to release n->note_mu to acquire - n->parent->note_mu. The parent could be disconnected and freed while - n->note_mu is not held. The n->disconnecting count handles this; the - operation acquires n->note_mu, increments n->disconnecting, and can then - release n->note_mu, and acquire n->parent->note_mu and n->note_mu is the - correct order. n->disconnecting!=0 indicates that a thread is already in - the processes of disconnecting n from n->parent. A thread freeing or - notifying the parent should not perform the disconnection of that child, but - should instead wait for the "children" list to become empty via - WAIT_FOR_NO_CHILDREN(). WAKEUP_NO_CHILDREN() should be used whenever this - condition could become true. */ - -/* Set the expiry time in *n to t */ -static void set_expiry_time (nsync_note n, nsync_time t) { - n->expiry_time = t; - n->expiry_time_valid = 1; -} - -/* Return a pointer to the note containing nsync_dll_element_ *e. */ -#define DLL_NOTE(e) ((nsync_note)((e)->container)) - -/* Return whether n->children is empty. Assumes n->note_mu held. */ -static int no_children (const void *v) { - return (nsync_dll_is_empty_ (((nsync_note)v)->children)); -} - -#define WAIT_FOR_NO_CHILDREN(pred_, n_) nsync_mu_wait (&(n_)->note_mu, &pred_, (n_), NULL) -#define WAKEUP_NO_CHILDREN(n_) do { } while (0) - -/* -// These lines can be used in place of those above if conditional critical -// sections have been removed from the source. -#define WAIT_FOR_NO_CHILDREN(pred_, n_) do { \ - while (!pred_ (n_)) { nsync_cv_wait (&(n_)->no_children_cv, &(n_)->note_mu); } \ - } while (0) -#define WAKEUP_NO_CHILDREN(n_) nsync_cv_broadcast (&(n_)->no_children_cv) -*/ - -/* Notify *n and all its descendants that are not already disconnnecting. - n->note_mu is held. May release and reacquire n->note_mu. - parent->note_mu is held if parent != NULL. */ -static void note_notify_child (nsync_note n, nsync_note parent) { - nsync_time t; - t = NOTIFIED_TIME (n); - if (nsync_time_cmp (t, nsync_time_zero) > 0) { - nsync_dll_element_ *p; - nsync_dll_element_ *next; - ATM_STORE_REL (&n->notified, 1); - while ((p = nsync_dll_first_ (n->waiters)) != NULL) { - struct nsync_waiter_s *nw = DLL_NSYNC_WAITER (p); - n->waiters = nsync_dll_remove_ (n->waiters, p); - ATM_STORE_REL (&nw->waiting, 0); - nsync_mu_semaphore_v (nw->sem); - } - for (p = nsync_dll_first_ (n->children); p != NULL; p = next) { - nsync_note child = DLL_NOTE (p); - next = nsync_dll_next_ (n->children, p); - nsync_mu_lock (&child->note_mu); - if (child->disconnecting == 0) { - note_notify_child (child, n); - } - nsync_mu_unlock (&child->note_mu); - } - WAIT_FOR_NO_CHILDREN (no_children, n); - if (parent != NULL) { - parent->children = nsync_dll_remove_ (parent->children, - &n->parent_child_link); - WAKEUP_NO_CHILDREN (parent); - n->parent = NULL; - } - } -} - -/* Notify *n and all its descendants that are not already disconnnecting. - No locks are held. */ -static void notify (nsync_note n) { - nsync_time t; - nsync_mu_lock (&n->note_mu); - t = NOTIFIED_TIME (n); - if (nsync_time_cmp (t, nsync_time_zero) > 0) { - nsync_note parent; - n->disconnecting++; - parent = n->parent; - if (parent != NULL && !nsync_mu_trylock (&parent->note_mu)) { - nsync_mu_unlock (&n->note_mu); - nsync_mu_lock (&parent->note_mu); - nsync_mu_lock (&n->note_mu); - } - note_notify_child (n, parent); - if (parent != NULL) { - nsync_mu_unlock (&parent->note_mu); - } - n->disconnecting--; - } - nsync_mu_unlock (&n->note_mu); -} - -/* Return the deadline by which *n is certain to be notified, - setting it to zero if it already has passed that time. - Requires n->note_mu not held on entry. - - Not static; used in sem_wait.c */ -nsync_time nsync_note_notified_deadline_ (nsync_note n) { - nsync_time ntime; - if (ATM_LOAD_ACQ (&n->notified) != 0) { - ntime = nsync_time_zero; - } else { - nsync_mu_lock (&n->note_mu); - ntime = NOTIFIED_TIME (n); - nsync_mu_unlock (&n->note_mu); - if (nsync_time_cmp (ntime, nsync_time_zero) > 0) { - if (nsync_time_cmp (ntime, nsync_time_now ()) <= 0) { - notify (n); - ntime = nsync_time_zero; - } - } - } - return (ntime); -} - -int nsync_note_is_notified (nsync_note n) { - int result; - IGNORE_RACES_START (); - result = (nsync_time_cmp (nsync_note_notified_deadline_ (n), nsync_time_zero) <= 0); - IGNORE_RACES_END (); - return (result); -} - -nsync_note nsync_note_new (nsync_note parent, - nsync_time abs_deadline) { - nsync_note n = (nsync_note) malloc (sizeof (*n)); - if (n != NULL) { - memset ((void *) n, 0, sizeof (*n)); - nsync_dll_init_ (&n->parent_child_link, n); - set_expiry_time (n, abs_deadline); - if (!nsync_note_is_notified (n) && parent != NULL) { - nsync_time parent_time; - nsync_mu_lock (&parent->note_mu); - parent_time = NOTIFIED_TIME (parent); - if (nsync_time_cmp (parent_time, abs_deadline) < 0) { - set_expiry_time (n, parent_time); - } - if (nsync_time_cmp (parent_time, nsync_time_zero) > 0) { - n->parent = parent; - parent->children = nsync_dll_make_last_in_list_ (parent->children, - &n->parent_child_link); - } - nsync_mu_unlock (&parent->note_mu); - } - } - return (n); -} - -void nsync_note_free (nsync_note n) { - nsync_note parent; - nsync_dll_element_ *p; - nsync_dll_element_ *next; - nsync_mu_lock (&n->note_mu); - n->disconnecting++; - ASSERT (nsync_dll_is_empty_ (n->waiters)); - parent = n->parent; - if (parent != NULL && !nsync_mu_trylock (&parent->note_mu)) { - nsync_mu_unlock (&n->note_mu); - nsync_mu_lock (&parent->note_mu); - nsync_mu_lock (&n->note_mu); - } - for (p = nsync_dll_first_ (n->children); p != NULL; p = next) { - nsync_note child = DLL_NOTE (p); - next = nsync_dll_next_ (n->children, p); - nsync_mu_lock (&child->note_mu); - if (child->disconnecting == 0) { - n->children = nsync_dll_remove_ (n->children, - &child->parent_child_link); - if (parent != NULL) { - child->parent = parent; - parent->children = nsync_dll_make_last_in_list_ ( - parent->children, &child->parent_child_link); - } else { - child->parent = NULL; - } - } - nsync_mu_unlock (&child->note_mu); - } - WAIT_FOR_NO_CHILDREN (no_children, n); - if (parent != NULL) { - parent->children = nsync_dll_remove_ (parent->children, - &n->parent_child_link); - WAKEUP_NO_CHILDREN (parent); - n->parent = NULL; - nsync_mu_unlock (&parent->note_mu); - } - n->disconnecting--; - nsync_mu_unlock (&n->note_mu); - free (n); -} - -void nsync_note_notify (nsync_note n) { - IGNORE_RACES_START (); - if (nsync_time_cmp (nsync_note_notified_deadline_ (n), nsync_time_zero) > 0) { - notify (n); - } - IGNORE_RACES_END (); -} - -int nsync_note_wait (nsync_note n, nsync_time abs_deadline) { - struct nsync_waitable_s waitable; - struct nsync_waitable_s *pwaitable = &waitable; - waitable.v = n; - waitable.funcs = &nsync_note_waitable_funcs; - return (nsync_wait_n (NULL, NULL, NULL, abs_deadline, 1, &pwaitable) == 0); -} - -nsync_time nsync_note_expiry (nsync_note n) { - return (n->expiry_time); -} - -static nsync_time note_ready_time (void *v, struct nsync_waiter_s *nw) { - return (nsync_note_notified_deadline_ ((nsync_note)v)); -} - -static int note_enqueue (void *v, struct nsync_waiter_s *nw) { - int waiting = 0; - nsync_note n = (nsync_note) v; - nsync_time ntime; - nsync_mu_lock (&n->note_mu); - ntime = NOTIFIED_TIME (n); - if (nsync_time_cmp (ntime, nsync_time_zero) > 0) { - n->waiters = nsync_dll_make_last_in_list_ (n->waiters, &nw->q); - ATM_STORE (&nw->waiting, 1); - waiting = 1; - } else { - ATM_STORE (&nw->waiting, 0); - waiting = 0; - } - nsync_mu_unlock (&n->note_mu); - return (waiting); -} - -static int note_dequeue (void *v, struct nsync_waiter_s *nw) { - int was_queued = 0; - nsync_note n = (nsync_note) v; - nsync_time ntime; - nsync_note_notified_deadline_ (n); - nsync_mu_lock (&n->note_mu); - ntime = NOTIFIED_TIME (n); - if (nsync_time_cmp (ntime, nsync_time_zero) > 0) { - n->waiters = nsync_dll_remove_ (n->waiters, &nw->q); - ATM_STORE (&nw->waiting, 0); - was_queued = 1; - } - nsync_mu_unlock (&n->note_mu); - return (was_queued); -} - -const struct nsync_waitable_funcs_s nsync_note_waitable_funcs = { - ¬e_ready_time, - ¬e_enqueue, - ¬e_dequeue -}; diff --git a/libc/thread/nsync_once.c b/libc/thread/nsync_once.c deleted file mode 100644 index 381f443c9..000000000 --- a/libc/thread/nsync_once.c +++ /dev/null @@ -1,150 +0,0 @@ -/*-*- mode:c;indent-tabs-mode:t;c-basic-offset:8;tab-width:8;coding:utf-8 -*-│ -│vi: set et ft=c ts=8 tw=8 fenc=utf-8 :vi│ -╞══════════════════════════════════════════════════════════════════════════════╡ -│ Copyright 2016 Google Inc. │ -│ │ -│ Licensed under the Apache License, Version 2.0 (the "License"); │ -│ you may not use this file except in compliance with the License. │ -│ You may obtain a copy of the License at │ -│ │ -│ http://www.apache.org/licenses/LICENSE-2.0 │ -│ │ -│ Unless required by applicable law or agreed to in writing, software │ -│ distributed under the License is distributed on an "AS IS" BASIS, │ -│ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. │ -│ See the License for the specific language governing permissions and │ -│ limitations under the License. │ -╚─────────────────────────────────────────────────────────────────────────────*/ -#include "third_party/nsync/atomic.h" -#include "third_party/nsync/atomic.internal.h" -#include "third_party/nsync/common.internal.h" -#include "third_party/nsync/dll.h" -#include "third_party/nsync/mu_semaphore.h" -#include "third_party/nsync/once.h" -#include "third_party/nsync/races.internal.h" -#include "third_party/nsync/wait_s.internal.h" - -asm(".ident\t\"\\n\\n\ -*NSYNC (Apache 2.0)\\n\ -Copyright 2016 Google, Inc.\\n\ -https://github.com/google/nsync\""); -// clang-format off - -/* An once_sync_s struct contains a lock, and a condition variable on which - threads may wait for an nsync_once to be initialized by another thread. - - A separate struct is used only to keep nsync_once small. - - A given nsync_once can be associated with any once_sync_s struct, but cannot - be associated with more than one. nsync_once instances are mapped to - once_sync_s instances by a trivial hashing scheme implemented by - NSYNC_ONCE_SYNC_(). - - The number of once_sync_s structs in the following array is greater than one - only to reduce the probability of contention if a great many distinct - nsync_once variables are initialized concurrently. */ -static struct once_sync_s { - nsync_mu once_mu; - nsync_cv once_cv; -} once_sync[64]; - -/* Return a pointer to the once_sync_s struct associated with the nsync_once *p. */ -#define NSYNC_ONCE_SYNC_(p) &once_sync[(((uintptr_t) (p)) / sizeof (*(p))) % \ - (sizeof (once_sync) / sizeof (once_sync[0]))] - -/* Implement nsync_run_once, nsync_run_once_arg, nsync_run_once_spin, or - nsync_run_once_arg_spin, chosen as described below. - - If s!=NULL, s is required to point to the once_sync_s associated with *once, - and the semantics of nsync_run_once or nsync_run_once_arg are provided. - If s==NULL, the semantics of nsync_run_once_spin, or nsync_run_once_arg_spin - are provided. - - If f!=NULL, the semantics of nsync_run_once or nsync_run_once_spin are - provided. Otherwise, farg is required to be non-NULL, and the semantics of - nsync_run_once_arg or nsync_run_once_arg_spin are provided. */ -static void nsync_run_once_impl (nsync_once *once, struct once_sync_s *s, - void (*f) (void), void (*farg) (void *arg), void *arg) { - uint32_t o = ATM_LOAD_ACQ (once); - if (o != 2) { - unsigned attempts = 0; - if (s != NULL) { - nsync_mu_lock (&s->once_mu); - } - while (o == 0 && !ATM_CAS_ACQ (once, 0, 1)) { - o = ATM_LOAD (once); - } - if (o == 0) { - if (s != NULL) { - nsync_mu_unlock (&s->once_mu); - } - if (f != NULL) { - (*f) (); - } else { - (*farg) (arg); - } - if (s != NULL) { - nsync_mu_lock (&s->once_mu); - nsync_cv_broadcast (&s->once_cv); - } - ATM_STORE_REL (once, 2); - } - while (ATM_LOAD_ACQ (once) != 2) { - if (s != NULL) { - nsync_time deadline; - if (attempts < 50) { - attempts += 10; - } - deadline = nsync_time_add (nsync_time_now (), nsync_time_ms (attempts)); - nsync_cv_wait_with_deadline (&s->once_cv, &s->once_mu, deadline, NULL); - } else { - attempts = nsync_spin_delay_ (attempts); - } - } - if (s != NULL) { - nsync_mu_unlock (&s->once_mu); - } - } -} - -void nsync_run_once (nsync_once *once, void (*f) (void)) { - uint32_t o; - IGNORE_RACES_START (); - o = ATM_LOAD_ACQ (once); - if (o != 2) { - struct once_sync_s *s = NSYNC_ONCE_SYNC_ (once); - nsync_run_once_impl (once, s, f, NULL, NULL); - } - IGNORE_RACES_END (); -} - -void nsync_run_once_arg (nsync_once *once, void (*farg) (void *arg), void *arg) { - uint32_t o; - IGNORE_RACES_START (); - o = ATM_LOAD_ACQ (once); - if (o != 2) { - struct once_sync_s *s = NSYNC_ONCE_SYNC_ (once); - nsync_run_once_impl (once, s, NULL, farg, arg); - } - IGNORE_RACES_END (); -} - -void nsync_run_once_spin (nsync_once *once, void (*f) (void)) { - uint32_t o; - IGNORE_RACES_START (); - o = ATM_LOAD_ACQ (once); - if (o != 2) { - nsync_run_once_impl (once, NULL, f, NULL, NULL); - } - IGNORE_RACES_END (); -} - -void nsync_run_once_arg_spin (nsync_once *once, void (*farg) (void *arg), void *arg) { - uint32_t o; - IGNORE_RACES_START (); - o = ATM_LOAD_ACQ (once); - if (o != 2) { - nsync_run_once_impl (once, NULL, NULL, farg, arg); - } - IGNORE_RACES_END (); -} diff --git a/libc/thread/nsync_sem_wait.c b/libc/thread/nsync_sem_wait.c deleted file mode 100644 index ae467a6a1..000000000 --- a/libc/thread/nsync_sem_wait.c +++ /dev/null @@ -1,85 +0,0 @@ -/*-*- mode:c;indent-tabs-mode:t;c-basic-offset:8;tab-width:8;coding:utf-8 -*-│ -│vi: set et ft=c ts=8 tw=8 fenc=utf-8 :vi│ -╞══════════════════════════════════════════════════════════════════════════════╡ -│ Copyright 2016 Google Inc. │ -│ │ -│ Licensed under the Apache License, Version 2.0 (the "License"); │ -│ you may not use this file except in compliance with the License. │ -│ You may obtain a copy of the License at │ -│ │ -│ http://www.apache.org/licenses/LICENSE-2.0 │ -│ │ -│ Unless required by applicable law or agreed to in writing, software │ -│ distributed under the License is distributed on an "AS IS" BASIS, │ -│ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. │ -│ See the License for the specific language governing permissions and │ -│ limitations under the License. │ -╚─────────────────────────────────────────────────────────────────────────────*/ -#include "libc/errno.h" -#include "third_party/nsync/atomic.h" -#include "third_party/nsync/atomic.internal.h" -#include "third_party/nsync/common.internal.h" -#include "third_party/nsync/dll.h" -#include "third_party/nsync/mu_semaphore.h" -#include "third_party/nsync/wait_s.internal.h" - -asm(".ident\t\"\\n\\n\ -*NSYNC (Apache 2.0)\\n\ -Copyright 2016 Google, Inc.\\n\ -https://github.com/google/nsync\""); -// clang-format off - -/* Wait until one of: - w->sem is non-zero----decrement it and return 0. - abs_deadline expires---return ETIMEDOUT. - cancel_note is non-NULL and *cancel_note becomes notified---return ECANCELED. */ -int nsync_sem_wait_with_cancel_ (waiter *w, nsync_time abs_deadline, - nsync_note cancel_note) { - int sem_outcome; - if (cancel_note == NULL) { - sem_outcome = nsync_mu_semaphore_p_with_deadline (&w->sem, abs_deadline); - } else { - nsync_time cancel_time; - cancel_time = nsync_note_notified_deadline_ (cancel_note); - sem_outcome = ECANCELED; - if (nsync_time_cmp (cancel_time, nsync_time_zero) > 0) { - struct nsync_waiter_s nw; - nw.tag = NSYNC_WAITER_TAG; - nw.sem = &w->sem; - nsync_dll_init_ (&nw.q, &nw); - ATM_STORE (&nw.waiting, 1); - nw.flags = 0; - nsync_mu_lock (&cancel_note->note_mu); - cancel_time = NOTIFIED_TIME (cancel_note); - if (nsync_time_cmp (cancel_time, nsync_time_zero) > 0) { - nsync_time local_abs_deadline; - int deadline_is_nearer = 0; - cancel_note->waiters = nsync_dll_make_last_in_list_ ( - cancel_note->waiters, &nw.q); - local_abs_deadline = cancel_time; - if (nsync_time_cmp (abs_deadline, cancel_time) < 0) { - local_abs_deadline = abs_deadline; - deadline_is_nearer = 1; - } - nsync_mu_unlock (&cancel_note->note_mu); - sem_outcome = nsync_mu_semaphore_p_with_deadline (&w->sem, - local_abs_deadline); - if (sem_outcome == ETIMEDOUT && !deadline_is_nearer) { - sem_outcome = ECANCELED; - nsync_note_notify (cancel_note); - } - nsync_mu_lock (&cancel_note->note_mu); - cancel_time = NOTIFIED_TIME (cancel_note); - if (nsync_time_cmp (cancel_time, - nsync_time_zero) > 0) { - cancel_note->waiters = nsync_dll_remove_ ( - cancel_note->waiters, &nw.q); - } - } - nsync_mu_unlock (&cancel_note->note_mu); - } - } - return (sem_outcome); -} - - diff --git a/libc/thread/nsync_sem_wait_no_note.c b/libc/thread/nsync_sem_wait_no_note.c deleted file mode 100644 index c9c3a156b..000000000 --- a/libc/thread/nsync_sem_wait_no_note.c +++ /dev/null @@ -1,33 +0,0 @@ -/*-*- mode:c;indent-tabs-mode:t;c-basic-offset:8;tab-width:8;coding:utf-8 -*-│ -│vi: set et ft=c ts=8 tw=8 fenc=utf-8 :vi│ -╞══════════════════════════════════════════════════════════════════════════════╡ -│ Copyright 2016 Google Inc. │ -│ │ -│ Licensed under the Apache License, Version 2.0 (the "License"); │ -│ you may not use this file except in compliance with the License. │ -│ You may obtain a copy of the License at │ -│ │ -│ http://www.apache.org/licenses/LICENSE-2.0 │ -│ │ -│ Unless required by applicable law or agreed to in writing, software │ -│ distributed under the License is distributed on an "AS IS" BASIS, │ -│ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. │ -│ See the License for the specific language governing permissions and │ -│ limitations under the License. │ -╚─────────────────────────────────────────────────────────────────────────────*/ -#include "third_party/nsync/common.internal.h" -#include "third_party/nsync/mu_semaphore.h" - -asm(".ident\t\"\\n\\n\ -*NSYNC (Apache 2.0)\\n\ -Copyright 2016 Google, Inc.\\n\ -https://github.com/google/nsync\""); -// clang-format off - -/* Wait until one of: - w->sem is non-zero----decrement it and return 0. - abs_deadline expires---return ETIMEDOUT. - Ignores cancel_note. */ -int nsync_sem_wait_with_cancel_ (waiter *w, nsync_time abs_deadline, nsync_note cancel_note) { - return (nsync_mu_semaphore_p_with_deadline (&w->sem, abs_deadline)); -} diff --git a/libc/thread/nsync_wait.c b/libc/thread/nsync_wait.c deleted file mode 100644 index de75ab165..000000000 --- a/libc/thread/nsync_wait.c +++ /dev/null @@ -1,109 +0,0 @@ -/*-*- mode:c;indent-tabs-mode:t;c-basic-offset:8;tab-width:8;coding:utf-8 -*-│ -│vi: set et ft=c ts=8 tw=8 fenc=utf-8 :vi│ -╞══════════════════════════════════════════════════════════════════════════════╡ -│ Copyright 2016 Google Inc. │ -│ │ -│ Licensed under the Apache License, Version 2.0 (the "License"); │ -│ you may not use this file except in compliance with the License. │ -│ You may obtain a copy of the License at │ -│ │ -│ http://www.apache.org/licenses/LICENSE-2.0 │ -│ │ -│ Unless required by applicable law or agreed to in writing, software │ -│ distributed under the License is distributed on an "AS IS" BASIS, │ -│ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. │ -│ See the License for the specific language governing permissions and │ -│ limitations under the License. │ -╚─────────────────────────────────────────────────────────────────────────────*/ -#include "libc/mem/mem.h" -#include "third_party/nsync/atomic.h" -#include "third_party/nsync/atomic.internal.h" -#include "third_party/nsync/common.internal.h" -#include "third_party/nsync/dll.h" -#include "third_party/nsync/mu_semaphore.h" -#include "third_party/nsync/races.internal.h" -#include "third_party/nsync/wait_s.internal.h" -#include "third_party/nsync/waiter.h" - -asm(".ident\t\"\\n\\n\ -*NSYNC (Apache 2.0)\\n\ -Copyright 2016 Google, Inc.\\n\ -https://github.com/google/nsync\""); -// clang-format off - -int nsync_wait_n (void *mu, void (*lock) (void *), void (*unlock) (void *), - nsync_time abs_deadline, - int count, struct nsync_waitable_s *waitable[]) { - int ready; - IGNORE_RACES_START (); - for (ready = 0; ready != count && - nsync_time_cmp ((*waitable[ready]->funcs->ready_time) ( - waitable[ready]->v, NULL), - nsync_time_zero) > 0; - ready++) { - } - if (ready == count && nsync_time_cmp (abs_deadline, nsync_time_zero) > 0) { - int i; - int unlocked = 0; - int j; - int enqueued = 1; - waiter *w = nsync_waiter_new_ (); - struct nsync_waiter_s nw_set[4]; - struct nsync_waiter_s *nw = nw_set; - if (count > (int) (sizeof (nw_set) / sizeof (nw_set[0]))) { - nw = (struct nsync_waiter_s *) malloc (count * sizeof (nw[0])); - } - for (i = 0; i != count && enqueued; i++) { - nw[i].tag = NSYNC_WAITER_TAG; - nw[i].sem = &w->sem; - nsync_dll_init_ (&nw[i].q, &nw[i]); - ATM_STORE (&nw[i].waiting, 0); - nw[i].flags = 0; - enqueued = (*waitable[i]->funcs->enqueue) (waitable[i]->v, &nw[i]); - } - - if (i == count) { - nsync_time min_ntime; - if (mu != NULL) { - (*unlock) (mu); - unlocked = 1; - } - do { - min_ntime = abs_deadline; - for (j = 0; j != count; j++) { - nsync_time ntime; - ntime = (*waitable[j]->funcs->ready_time) ( - waitable[j]->v, &nw[j]); - if (nsync_time_cmp (ntime, min_ntime) < 0) { - min_ntime = ntime; - } - } - } while (nsync_time_cmp (min_ntime, nsync_time_zero) > 0 && - nsync_mu_semaphore_p_with_deadline (&w->sem, - min_ntime) == 0); - } - - /* An attempt was made above to enqueue waitable[0..i-1]. - Dequeue any that are still enqueued, and remember the index - of the first ready (i.e., not still enqueued) object, if any. */ - for (j = 0; j != i; j++) { - int was_still_enqueued = - (*waitable[j]->funcs->dequeue) (waitable[j]->v, &nw[j]); - if (!was_still_enqueued && ready == count) { - ready = j; - } - } - - if (nw != nw_set) { - free (nw); - } - nsync_waiter_free_ (w); - if (unlocked) { - (*lock) (mu); - } - } - IGNORE_RACES_END (); - return (ready); -} - - diff --git a/libc/thread/posixthread.internal.h b/libc/thread/posixthread.internal.h index bfbf41e9a..7a51695dc 100644 --- a/libc/thread/posixthread.internal.h +++ b/libc/thread/posixthread.internal.h @@ -1,8 +1,8 @@ #ifndef COSMOPOLITAN_LIBC_THREAD_POSIXTHREAD_INTERNAL_H_ #define COSMOPOLITAN_LIBC_THREAD_POSIXTHREAD_INTERNAL_H_ #include "libc/calls/struct/sched_param.h" +#include "libc/intrin/pthread.h" #include "libc/runtime/runtime.h" -#include "libc/thread/thread.h" #if !(__ASSEMBLER__ + __LINKER__ + 0) COSMOPOLITAN_C_START_ @@ -62,26 +62,18 @@ struct PosixThread { int *ctid; char *tls; char *tib; - char *altstack; _Atomic(enum PosixThreadStatus) status; jmp_buf exiter; pthread_attr_t attr; }; -hidden extern pthread_spinlock_t _pthread_keys_lock; -hidden extern uint64_t _pthread_key_usage[(PTHREAD_KEYS_MAX + 63) / 64]; -hidden extern pthread_key_dtor _pthread_key_dtor[PTHREAD_KEYS_MAX]; -hidden extern _Thread_local void *_pthread_keys[PTHREAD_KEYS_MAX]; - -int _pthread_reschedule(struct PosixThread *) hidden; -int _pthread_setschedparam_freebsd(int, int, const struct sched_param *) hidden; void _pthread_free(struct PosixThread *) hidden; -void _pthread_ungarbage(void) hidden; void _pthread_wait(struct PosixThread *) hidden; +int _pthread_reschedule(struct PosixThread *) hidden; void _pthread_zombies_add(struct PosixThread *) hidden; void _pthread_zombies_decimate(void) hidden; void _pthread_zombies_harvest(void) hidden; -void _pthread_key_destruct(void *[PTHREAD_KEYS_MAX]); +int _pthread_setschedparam_freebsd(int, int, const struct sched_param *); COSMOPOLITAN_C_END_ #endif /* !(__ASSEMBLER__ + __LINKER__ + 0) */ diff --git a/libc/thread/pthread_cond_timedwait.c b/libc/thread/pthread_cond_timedwait.c index 31ecf88ba..1245fb77c 100644 --- a/libc/thread/pthread_cond_timedwait.c +++ b/libc/thread/pthread_cond_timedwait.c @@ -16,12 +16,14 @@ │ TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR │ │ PERFORMANCE OF THIS SOFTWARE. │ ╚─────────────────────────────────────────────────────────────────────────────*/ +#include "libc/assert.h" +#include "libc/calls/calls.h" +#include "libc/calls/struct/timespec.h" #include "libc/errno.h" -#include "libc/thread/thread.h" -#include "libc/thread/thread2.h" -#include "third_party/nsync/common.internal.h" -#include "third_party/nsync/cv.h" -#include "third_party/nsync/time.h" +#include "libc/intrin/atomic.h" +#include "libc/intrin/futex.internal.h" +#include "libc/intrin/pthread.h" +#include "libc/intrin/pthread2.h" /** * Waits for condition with optional time limit, e.g. @@ -46,13 +48,42 @@ */ int pthread_cond_timedwait(pthread_cond_t *cond, pthread_mutex_t *mutex, const struct timespec *abstime) { + int rc, err, seq; + struct timespec now, rel, *tsp; + if (abstime && !(0 <= abstime->tv_nsec && abstime->tv_nsec < 1000000000)) { + assert(!"bad abstime"); return EINVAL; } - if (mutex->_type != PTHREAD_MUTEX_NORMAL) { - nsync_panic_("pthread cond needs normal mutex\n"); + + if ((err = pthread_mutex_unlock(mutex))) { + return err; } - return nsync_cv_wait_with_deadline( - (nsync_cv *)cond, (nsync_mu *)mutex, - abstime ? *abstime : nsync_time_no_deadline, 0); + + atomic_fetch_add(&cond->waits, 1); + + rc = 0; + seq = atomic_load_explicit(&cond->seq, memory_order_relaxed); + do { + if (!abstime) { + tsp = 0; + } else { + now = _timespec_mono(); + if (_timespec_gte(now, *abstime)) { + rc = ETIMEDOUT; + break; + } + rel = _timespec_sub(*abstime, now); + tsp = &rel; + } + _futex_wait(&cond->seq, seq, cond->pshared, tsp); + } while (seq == atomic_load_explicit(&cond->seq, memory_order_relaxed)); + + atomic_fetch_sub(&cond->waits, 1); + + if ((err = pthread_mutex_lock(mutex))) { + return err; + } + + return rc; } diff --git a/libc/thread/pthread_cond_wait.c b/libc/thread/pthread_cond_wait.c index 30920d280..1f48fba32 100644 --- a/libc/thread/pthread_cond_wait.c +++ b/libc/thread/pthread_cond_wait.c @@ -16,19 +16,24 @@ │ TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR │ │ PERFORMANCE OF THIS SOFTWARE. │ ╚─────────────────────────────────────────────────────────────────────────────*/ -#include "libc/thread/thread.h" -#include "libc/thread/thread2.h" +#include "libc/intrin/pthread2.h" /** * Waits for condition, e.g. * * pthread_cond_t cond = PTHREAD_COND_INITIALIZER; * pthread_mutex_t lock = PTHREAD_MUTEX_INITIALIZER; - * // ... + * + * // waiting threads * pthread_mutex_lock(&lock); * pthread_cond_wait(&cond, &lock); * pthread_mutex_unlock(&lock); * + * // notifying thread + * pthread_mutex_lock(&lock); + * pthread_cond_broadcast(&cond); + * pthread_mutex_unlock(&lock); + * * @param mutex needs to be held by thread when calling this function * @return 0 on success, or errno on error * @raise EPERM if `mutex` is `PTHREAD_MUTEX_ERRORCHECK` and the lock diff --git a/libc/thread/pthread_create.c b/libc/thread/pthread_create.c index 5ae34c19c..a08471e7e 100644 --- a/libc/thread/pthread_create.c +++ b/libc/thread/pthread_create.c @@ -19,34 +19,30 @@ #include "libc/assert.h" #include "libc/calls/calls.h" #include "libc/calls/sched-sysv.internal.h" -#include "libc/calls/struct/sigaltstack.h" #include "libc/calls/syscall-sysv.internal.h" #include "libc/dce.h" #include "libc/errno.h" #include "libc/intrin/asan.internal.h" #include "libc/intrin/atomic.h" #include "libc/intrin/bits.h" -#include "libc/intrin/kprintf.h" +#include "libc/intrin/pthread.h" +#include "libc/intrin/wait0.internal.h" #include "libc/intrin/weaken.h" -#include "libc/log/internal.h" #include "libc/macros.internal.h" #include "libc/mem/mem.h" #include "libc/nexgen32e/gc.internal.h" +#include "libc/nexgen32e/gettls.h" +#include "libc/nexgen32e/threaded.h" #include "libc/runtime/runtime.h" #include "libc/runtime/stack.h" #include "libc/sysv/consts/clone.h" #include "libc/sysv/consts/map.h" #include "libc/sysv/consts/prot.h" -#include "libc/sysv/consts/ss.h" #include "libc/sysv/errfuns.h" +#include "libc/thread/internal.h" #include "libc/thread/posixthread.internal.h" #include "libc/thread/spawn.h" #include "libc/thread/thread.h" -#include "libc/thread/tls.h" -#include "libc/thread/wait0.internal.h" - -STATIC_YOINK("nsync_mu_lock"); -STATIC_YOINK("nsync_mu_unlock"); #define MAP_ANON_OPENBSD 0x1000 #define MAP_STACK_OPENBSD 0x4000 @@ -60,12 +56,7 @@ void _pthread_free(struct PosixThread *pt) { if (pt->ownstack && // pt->attr.stackaddr && // pt->attr.stackaddr != MAP_FAILED) { - if (munmap(pt->attr.stackaddr, pt->attr.stacksize)) { - notpossible; - } - } - if (pt->altstack) { - free(pt->altstack); + munmap(&pt->attr.stackaddr, pt->attr.stacksize); } free(pt); } @@ -73,26 +64,17 @@ void _pthread_free(struct PosixThread *pt) { static int PosixThread(void *arg, int tid) { struct PosixThread *pt = arg; enum PosixThreadStatus status; - struct sigaltstack ss; - if (pt->altstack) { - ss.ss_flags = 0; - ss.ss_size = SIGSTKSZ; - ss.ss_sp = pt->altstack; - if (sigaltstack(&ss, 0)) { - notpossible; - } - } if (pt->attr.inheritsched == PTHREAD_EXPLICIT_SCHED) { _pthread_reschedule(pt); } if (!setjmp(pt->exiter)) { - __get_tls()->tib_pthread = (pthread_t)pt; + ((cthread_t)__get_tls())->pthread = (pthread_t)pt; pt->rc = pt->start_routine(pt->arg); } if (weaken(_pthread_key_destruct)) { weaken(_pthread_key_destruct)(0); } - _pthread_ungarbage(); + cthread_ungarbage(); if (atomic_load_explicit(&pt->status, memory_order_acquire) == kPosixThreadDetached) { atomic_store_explicit(&pt->status, kPosixThreadZombie, @@ -187,7 +169,7 @@ int pthread_create(pthread_t *thread, const pthread_attr_t *attr, void *(*start_routine)(void *), void *arg) { int rc, e = errno; struct PosixThread *pt; - __require_tls(); + TlsIsRequired(); _pthread_zombies_decimate(); // create posix thread object @@ -275,11 +257,6 @@ int pthread_create(pthread_t *thread, const pthread_attr_t *attr, } } - // setup signal handler stack - if (_wantcrashreports && !IsWindows()) { - pt->altstack = malloc(SIGSTKSZ); - } - // set initial status switch (pt->attr.detachstate) { case PTHREAD_CREATE_JOINABLE: diff --git a/libc/thread/pthread_detach.c b/libc/thread/pthread_detach.c index 9fbd94bce..c3aa460dd 100644 --- a/libc/thread/pthread_detach.c +++ b/libc/thread/pthread_detach.c @@ -17,7 +17,7 @@ │ PERFORMANCE OF THIS SOFTWARE. │ ╚─────────────────────────────────────────────────────────────────────────────*/ #include "libc/intrin/atomic.h" -#include "libc/thread/thread.h" +#include "libc/intrin/pthread.h" #include "libc/macros.internal.h" #include "libc/mem/mem.h" #include "libc/thread/posixthread.internal.h" diff --git a/libc/thread/pthread_equal.c b/libc/thread/pthread_equal.c index 6941eb9d3..c7e3afcce 100644 --- a/libc/thread/pthread_equal.c +++ b/libc/thread/pthread_equal.c @@ -16,7 +16,7 @@ │ TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR │ │ PERFORMANCE OF THIS SOFTWARE. │ ╚─────────────────────────────────────────────────────────────────────────────*/ -#include "libc/thread/thread.h" +#include "libc/intrin/pthread.h" #include "libc/thread/posixthread.internal.h" /** diff --git a/libc/thread/pthread_exit.c b/libc/thread/pthread_exit.c index de4d06fd0..842dc7e7f 100644 --- a/libc/thread/pthread_exit.c +++ b/libc/thread/pthread_exit.c @@ -16,11 +16,12 @@ │ TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR │ │ PERFORMANCE OF THIS SOFTWARE. │ ╚─────────────────────────────────────────────────────────────────────────────*/ +#include "libc/intrin/pthread.h" +#include "libc/nexgen32e/gettls.h" #include "libc/runtime/gc.h" #include "libc/thread/posixthread.internal.h" #include "libc/thread/spawn.h" #include "libc/thread/thread.h" -#include "libc/thread/tls.h" /** * Terminates current POSIX thread. @@ -36,7 +37,7 @@ */ wontreturn void pthread_exit(void *rc) { struct PosixThread *pt; - if ((pt = (struct PosixThread *)__get_tls()->tib_pthread)) { + if ((pt = (struct PosixThread *)((cthread_t)__get_tls())->pthread)) { pt->rc = rc; _gclongjmp(pt->exiter, 1); } else { diff --git a/libc/thread/pthread_getaffinity_np.c b/libc/thread/pthread_getaffinity_np.c index 0133fe20c..35064bbf5 100644 --- a/libc/thread/pthread_getaffinity_np.c +++ b/libc/thread/pthread_getaffinity_np.c @@ -19,6 +19,7 @@ #include "libc/calls/struct/cpuset.h" #include "libc/errno.h" #include "libc/thread/posixthread.internal.h" +#include "libc/thread/thread.h" /** * Gets CPU affinity for thread. diff --git a/libc/thread/pthread_getattr_np.c b/libc/thread/pthread_getattr_np.c index 479f4371c..878d7ecd9 100644 --- a/libc/thread/pthread_getattr_np.c +++ b/libc/thread/pthread_getattr_np.c @@ -16,6 +16,7 @@ │ TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR │ │ PERFORMANCE OF THIS SOFTWARE. │ ╚─────────────────────────────────────────────────────────────────────────────*/ +#include "libc/intrin/pthread.h" #include "libc/str/str.h" #include "libc/thread/posixthread.internal.h" #include "libc/thread/thread.h" diff --git a/libc/thread/pthread_getname_np.c b/libc/thread/pthread_getname_np.c index 413cb4c0a..a306fcd79 100644 --- a/libc/thread/pthread_getname_np.c +++ b/libc/thread/pthread_getname_np.c @@ -27,6 +27,7 @@ #include "libc/sysv/consts/o.h" #include "libc/sysv/consts/pr.h" #include "libc/thread/posixthread.internal.h" +#include "libc/thread/thread.h" /** * Gets name of thread registered with system, e.g. diff --git a/libc/thread/pthread_getschedparam.c b/libc/thread/pthread_getschedparam.c index f382a59dc..b98ef8873 100644 --- a/libc/thread/pthread_getschedparam.c +++ b/libc/thread/pthread_getschedparam.c @@ -16,8 +16,8 @@ │ TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR │ │ PERFORMANCE OF THIS SOFTWARE. │ ╚─────────────────────────────────────────────────────────────────────────────*/ +#include "libc/intrin/pthread2.h" #include "libc/thread/posixthread.internal.h" -#include "libc/thread/thread2.h" /** * Gets most recently set scheduling of thread. diff --git a/libc/intrin/pthread_getspecific.c b/libc/thread/pthread_getspecific.c similarity index 95% rename from libc/intrin/pthread_getspecific.c rename to libc/thread/pthread_getspecific.c index 404dcb1fd..0aa0771af 100644 --- a/libc/intrin/pthread_getspecific.c +++ b/libc/thread/pthread_getspecific.c @@ -16,7 +16,9 @@ │ TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR │ │ PERFORMANCE OF THIS SOFTWARE. │ ╚─────────────────────────────────────────────────────────────────────────────*/ -#include "libc/thread/posixthread.internal.h" +#include "libc/errno.h" +#include "libc/nexgen32e/gettls.h" +#include "libc/thread/internal.h" #include "libc/thread/thread.h" /** diff --git a/libc/thread/pthread_getthreadid_np.c b/libc/thread/pthread_getthreadid_np.c index 9ba68cb87..e3ceefcdc 100644 --- a/libc/thread/pthread_getthreadid_np.c +++ b/libc/thread/pthread_getthreadid_np.c @@ -17,7 +17,7 @@ │ PERFORMANCE OF THIS SOFTWARE. │ ╚─────────────────────────────────────────────────────────────────────────────*/ #include "libc/calls/calls.h" -#include "libc/thread/thread.h" +#include "libc/intrin/pthread.h" /** * Returns thread id of current POSIX thread. diff --git a/libc/thread/pthread_getunique_np.c b/libc/thread/pthread_getunique_np.c index d3bf75032..8e8b58637 100644 --- a/libc/thread/pthread_getunique_np.c +++ b/libc/thread/pthread_getunique_np.c @@ -17,6 +17,7 @@ │ PERFORMANCE OF THIS SOFTWARE. │ ╚─────────────────────────────────────────────────────────────────────────────*/ #include "libc/thread/posixthread.internal.h" +#include "libc/thread/thread.h" /** * Returns thread id of POSIX thread. diff --git a/libc/thread/pthread_join.c b/libc/thread/pthread_join.c index ea9535bfd..8ae82d909 100644 --- a/libc/thread/pthread_join.c +++ b/libc/thread/pthread_join.c @@ -18,7 +18,7 @@ ╚─────────────────────────────────────────────────────────────────────────────*/ #include "libc/assert.h" #include "libc/errno.h" -#include "libc/thread/thread.h" +#include "libc/intrin/pthread.h" #include "libc/macros.internal.h" #include "libc/mem/mem.h" #include "libc/thread/posixthread.internal.h" diff --git a/libc/intrin/pthread_key_create.c b/libc/thread/pthread_key_create.c similarity index 91% rename from libc/intrin/pthread_key_create.c rename to libc/thread/pthread_key_create.c index e7c7a9c98..6d0eb1a45 100644 --- a/libc/intrin/pthread_key_create.c +++ b/libc/thread/pthread_key_create.c @@ -18,29 +18,26 @@ ╚─────────────────────────────────────────────────────────────────────────────*/ #include "libc/errno.h" #include "libc/nexgen32e/bsr.h" +#include "libc/nexgen32e/gettls.h" #include "libc/runtime/runtime.h" -#include "libc/thread/posixthread.internal.h" +#include "libc/thread/internal.h" #include "libc/thread/thread.h" -#include "libc/thread/tls.h" /** * Allocates TLS slot. */ int pthread_key_create(pthread_key_t *key, pthread_key_dtor dtor) { - int i, j, rc = EAGAIN; - pthread_spin_lock(&_pthread_keys_lock); + int i, j; for (i = 0; i < (PTHREAD_KEYS_MAX + 63) / 64; ++i) { if (~_pthread_key_usage[i]) { j = bsrl(~_pthread_key_usage[i]); _pthread_key_usage[i] |= 1ul << j; _pthread_key_dtor[i * 64 + j] = dtor; *key = i * 64 + j; - rc = 0; - break; + return 0; } } - pthread_spin_unlock(&_pthread_keys_lock); - return rc; + return EAGAIN; } static textexit void _pthread_key_atexit(void) { diff --git a/libc/intrin/pthread_key_delete.c b/libc/thread/pthread_key_delete.c similarity index 91% rename from libc/intrin/pthread_key_delete.c rename to libc/thread/pthread_key_delete.c index b78120c21..998e78ac0 100644 --- a/libc/intrin/pthread_key_delete.c +++ b/libc/thread/pthread_key_delete.c @@ -17,22 +17,18 @@ │ PERFORMANCE OF THIS SOFTWARE. │ ╚─────────────────────────────────────────────────────────────────────────────*/ #include "libc/errno.h" -#include "libc/thread/posixthread.internal.h" +#include "libc/thread/internal.h" #include "libc/thread/thread.h" /** * Deletes TLS slot. */ int pthread_key_delete(pthread_key_t key) { - int rc; - pthread_spin_lock(&_pthread_keys_lock); if (key < PTHREAD_KEYS_MAX) { _pthread_key_usage[key / 64] &= ~(1ul << (key % 64)); _pthread_key_dtor[key] = 0; - rc = 0; + return 0; } else { - rc = EINVAL; + return EINVAL; } - pthread_spin_unlock(&_pthread_keys_lock); - return rc; } diff --git a/libc/intrin/pthread_key_destruct.c b/libc/thread/pthread_key_destruct.c similarity index 89% rename from libc/intrin/pthread_key_destruct.c rename to libc/thread/pthread_key_destruct.c index cfad22162..f9eb6cd0b 100644 --- a/libc/intrin/pthread_key_destruct.c +++ b/libc/thread/pthread_key_destruct.c @@ -17,17 +17,14 @@ │ PERFORMANCE OF THIS SOFTWARE. │ ╚─────────────────────────────────────────────────────────────────────────────*/ #include "libc/nexgen32e/bsr.h" -#include "libc/thread/posixthread.internal.h" +#include "libc/thread/internal.h" #include "libc/thread/thread.h" -#include "libc/thread/tls.h" void _pthread_key_destruct(void *key[PTHREAD_KEYS_MAX]) { int i, j; uint64_t x; void *value; pthread_key_dtor dtor; - if (!__tls_enabled) return; - pthread_spin_lock(&_pthread_keys_lock); if (!key) key = _pthread_keys; StartOver: for (i = 0; i < (PTHREAD_KEYS_MAX + 63) / 64; ++i) { @@ -36,13 +33,10 @@ StartOver: j = bsrl(x); if ((value = key[i * 64 + j]) && (dtor = _pthread_key_dtor[i * 64 + j])) { key[i * 64 + j] = 0; - pthread_spin_unlock(&_pthread_keys_lock); dtor(value); - pthread_spin_lock(&_pthread_keys_lock); goto StartOver; } x &= ~(1ul << j); } } - pthread_spin_unlock(&_pthread_keys_lock); } diff --git a/libc/thread/pthread_reschedule.c b/libc/thread/pthread_reschedule.c index c3ad753aa..2b7cc3664 100644 --- a/libc/thread/pthread_reschedule.c +++ b/libc/thread/pthread_reschedule.c @@ -22,6 +22,7 @@ #include "libc/errno.h" #include "libc/sysv/errfuns.h" #include "libc/thread/posixthread.internal.h" +#include "libc/thread/thread.h" int _pthread_reschedule(struct PosixThread *pt) { int rc, e = errno; diff --git a/libc/thread/pthread_self.c b/libc/thread/pthread_self.c index 358679b61..0212ecb83 100644 --- a/libc/thread/pthread_self.c +++ b/libc/thread/pthread_self.c @@ -16,12 +16,13 @@ │ TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR │ │ PERFORMANCE OF THIS SOFTWARE. │ ╚─────────────────────────────────────────────────────────────────────────────*/ +#include "libc/intrin/pthread.h" +#include "libc/nexgen32e/gettls.h" #include "libc/thread/thread.h" -#include "libc/thread/tls.h" /** * Returns current POSIX thread. */ pthread_t pthread_self(void) { - return __get_tls()->tib_pthread; + return ((cthread_t)__get_tls())->pthread; } diff --git a/libc/thread/pthread_setaffinity_np.c b/libc/thread/pthread_setaffinity_np.c index 9605d056e..28a67ceab 100644 --- a/libc/thread/pthread_setaffinity_np.c +++ b/libc/thread/pthread_setaffinity_np.c @@ -19,6 +19,7 @@ #include "libc/calls/struct/cpuset.h" #include "libc/errno.h" #include "libc/thread/posixthread.internal.h" +#include "libc/thread/thread.h" /** * Asks kernel to only schedule thread on particular CPUs. diff --git a/libc/thread/pthread_setname_np.c b/libc/thread/pthread_setname_np.c index 5452299ae..0663b05db 100644 --- a/libc/thread/pthread_setname_np.c +++ b/libc/thread/pthread_setname_np.c @@ -27,6 +27,7 @@ #include "libc/sysv/consts/o.h" #include "libc/sysv/consts/pr.h" #include "libc/thread/posixthread.internal.h" +#include "libc/thread/thread.h" /** * Registers custom name of thread with system, e.g. diff --git a/libc/thread/pthread_setschedparam.c b/libc/thread/pthread_setschedparam.c index e0d2fce57..4d44b2093 100644 --- a/libc/thread/pthread_setschedparam.c +++ b/libc/thread/pthread_setschedparam.c @@ -17,8 +17,8 @@ │ PERFORMANCE OF THIS SOFTWARE. │ ╚─────────────────────────────────────────────────────────────────────────────*/ #include "libc/errno.h" +#include "libc/intrin/pthread2.h" #include "libc/thread/posixthread.internal.h" -#include "libc/thread/thread2.h" /** * Changes scheduling of thread, e.g. diff --git a/libc/thread/pthread_setschedparam_freebsd.c b/libc/thread/pthread_setschedparam_freebsd.c index 4dfb10fe8..2a54cceb4 100644 --- a/libc/thread/pthread_setschedparam_freebsd.c +++ b/libc/thread/pthread_setschedparam_freebsd.c @@ -20,6 +20,7 @@ #include "libc/sysv/consts/sched.h" #include "libc/thread/freebsd.internal.h" #include "libc/thread/posixthread.internal.h" +#include "libc/thread/thread.h" #define RTP_SET_FREEBSD 1 #define PRI_REALTIME_FREEBSD 2 diff --git a/libc/intrin/pthread_setspecific.c b/libc/thread/pthread_setspecific.c similarity index 96% rename from libc/intrin/pthread_setspecific.c rename to libc/thread/pthread_setspecific.c index 18a663150..8d5b699d8 100644 --- a/libc/intrin/pthread_setspecific.c +++ b/libc/thread/pthread_setspecific.c @@ -17,7 +17,8 @@ │ PERFORMANCE OF THIS SOFTWARE. │ ╚─────────────────────────────────────────────────────────────────────────────*/ #include "libc/errno.h" -#include "libc/thread/posixthread.internal.h" +#include "libc/nexgen32e/gettls.h" +#include "libc/thread/internal.h" #include "libc/thread/thread.h" /** diff --git a/libc/thread/pthread_sigmask.c b/libc/thread/pthread_sigmask.c index 8c0340e65..20d87e802 100644 --- a/libc/thread/pthread_sigmask.c +++ b/libc/thread/pthread_sigmask.c @@ -17,6 +17,7 @@ │ PERFORMANCE OF THIS SOFTWARE. │ ╚─────────────────────────────────────────────────────────────────────────────*/ #include "libc/calls/struct/sigset.h" +#include "libc/thread/thread.h" /** * Examines and/or changes blocked signals on current thread. diff --git a/libc/thread/spawn.c b/libc/thread/spawn.c index 6ec8a144d..e58f76a59 100644 --- a/libc/thread/spawn.c +++ b/libc/thread/spawn.c @@ -17,8 +17,11 @@ │ PERFORMANCE OF THIS SOFTWARE. │ ╚─────────────────────────────────────────────────────────────────────────────*/ #include "libc/calls/calls.h" +#include "libc/intrin/kprintf.h" +#include "libc/intrin/wait0.internal.h" #include "libc/macros.internal.h" #include "libc/mem/mem.h" +#include "libc/nexgen32e/threaded.h" #include "libc/runtime/internal.h" #include "libc/runtime/runtime.h" #include "libc/runtime/stack.h" @@ -28,10 +31,8 @@ #include "libc/sysv/consts/map.h" #include "libc/sysv/consts/prot.h" #include "libc/sysv/errfuns.h" -#include "libc/thread/posixthread.internal.h" #include "libc/thread/spawn.h" -#include "libc/thread/tls.h" -#include "libc/thread/wait0.internal.h" +#include "libc/thread/thread.h" /** * @fileoverview Simple threading API @@ -51,8 +52,8 @@ #define _TLSZ ((intptr_t)_tls_size) #define _TLDZ ((intptr_t)_tdata_size) -#define _TIBZ sizeof(struct CosmoTib) -#define _MEMZ ROUNDUP(_TLSZ + _TIBZ, alignof(struct CosmoTib)) +#define _TIBZ sizeof(struct cthread_descriptor_t) +#define _MEMZ ROUNDUP(_TLSZ + _TIBZ, alignof(struct cthread_descriptor_t)) struct spawner { int (*fun)(void *, int); @@ -63,7 +64,7 @@ static int Spawner(void *arg, int tid) { int rc; struct spawner *spawner = arg; rc = spawner->fun(spawner->arg, tid); - _pthread_ungarbage(); + cthread_ungarbage(); free(spawner); return 0; } @@ -93,7 +94,7 @@ static int Spawner(void *arg, int tid) { int _spawn(int fun(void *, int), void *arg, struct spawn *opt_out_thread) { struct spawn *th, ths; struct spawner *spawner; - __require_tls(); + TlsIsRequired(); if (!fun) return einval(); // we need to to clobber the output memory before calling clone, since diff --git a/libc/thread/thread.h b/libc/thread/thread.h index ab97823ef..1d5b5cc0f 100644 --- a/libc/thread/thread.h +++ b/libc/thread/thread.h @@ -1,194 +1,32 @@ #ifndef COSMOPOLITAN_LIBC_THREAD_THREAD_H_ #define COSMOPOLITAN_LIBC_THREAD_THREAD_H_ - -#define PTHREAD_KEYS_MAX 64 -#define PTHREAD_STACK_MIN FRAMESIZE -#define PTHREAD_DESTRUCTOR_ITERATIONS 4 - -#define PTHREAD_BARRIER_SERIAL_THREAD 31337 - -#define PTHREAD_MUTEX_DEFAULT 0 -#define PTHREAD_MUTEX_NORMAL 0 -#define PTHREAD_MUTEX_RECURSIVE 1 -#define PTHREAD_MUTEX_ERRORCHECK 2 -#define PTHREAD_MUTEX_STALLED 0 -#define PTHREAD_MUTEX_ROBUST 1 - -#define PTHREAD_PROCESS_PRIVATE 0 -#define PTHREAD_PROCESS_SHARED 1 - -#define PTHREAD_CREATE_JOINABLE 0 -#define PTHREAD_CREATE_DETACHED 1 - -#define PTHREAD_INHERIT_SCHED 0 -#define PTHREAD_EXPLICIT_SCHED 1 - +#include "libc/intrin/pthread.h" +#include "libc/runtime/runtime.h" +#include "libc/str/locale.h" #if !(__ASSEMBLER__ + __LINKER__ + 0) COSMOPOLITAN_C_START_ -#define PTHREAD_ONCE_INIT _PTHREAD_INIT -#define PTHREAD_COND_INITIALIZER _PTHREAD_INIT -#define PTHREAD_RWLOCK_INITIALIZER _PTHREAD_INIT -#define PTHREAD_MUTEX_INITIALIZER _PTHREAD_INIT -#define _PTHREAD_INIT \ - { 0 } +struct Ftrace { /* 16 */ + bool once; /* 0 */ + bool noreentry; /* 1 */ + int skew; /* 4 */ + int64_t lastaddr; /* 8 */ +}; -typedef uintptr_t pthread_t; -typedef int pthread_id_np_t; -typedef char pthread_condattr_t; -typedef char pthread_rwlockattr_t; -typedef char pthread_barrierattr_t; -typedef unsigned pthread_key_t; -typedef void (*pthread_key_dtor)(void *); +struct cthread_descriptor_t { + struct cthread_descriptor_t *self; /* 0x00 */ + struct Ftrace ftrace; /* 0x08 */ + void *garbages; /* 0x18 */ + locale_t locale; /* 0x20 */ + pthread_t pthread; /* 0x28 */ + struct cthread_descriptor_t *self2; /* 0x30 */ + int32_t tid; /* 0x38 */ + int32_t err; /* 0x3c */ +}; -typedef struct pthread_once_s { - _Atomic(uint32_t) _lock; -} pthread_once_t; +typedef struct cthread_descriptor_t *cthread_t; -typedef struct pthread_spinlock_s { - _Atomic(char) _lock; -} pthread_spinlock_t; - -typedef struct pthread_mutex_s { - _Atomic(int) _lock; - void *_nsync; - char _type; - char _pshared; -} pthread_mutex_t; - -typedef struct pthread_mutexattr_s { - char _type; - char _pshared; -} pthread_mutexattr_t; - -typedef struct pthread_cond_s { - void *_nsync[2]; -} pthread_cond_t; - -typedef struct pthread_rwlock_s { - void *_nsync[2]; - char _iswrite; -} pthread_rwlock_t; - -typedef struct pthread_barrier_s { - void *_nsync; -} pthread_barrier_t; - -typedef struct pthread_attr_s { - char detachstate; - char inheritsched; - int schedparam; - int schedpolicy; - int scope; - unsigned guardsize; - unsigned stacksize; - char *stackaddr; -} pthread_attr_t; - -int pthread_yield(void); -void pthread_exit(void *) wontreturn; -pthread_t pthread_self(void) pureconst; -pthread_id_np_t pthread_getthreadid_np(void); -int64_t pthread_getunique_np(pthread_t); -int pthread_setname_np(pthread_t, const char *); -int pthread_getname_np(pthread_t, char *, size_t); -int pthread_getattr_np(pthread_t, pthread_attr_t *); -int pthread_attr_init(pthread_attr_t *); -int pthread_attr_destroy(pthread_attr_t *); -int pthread_attr_getdetachstate(const pthread_attr_t *, int *); -int pthread_attr_setdetachstate(pthread_attr_t *, int); -int pthread_attr_getguardsize(const pthread_attr_t *, size_t *); -int pthread_attr_setguardsize(pthread_attr_t *, size_t); -int pthread_attr_getinheritsched(const pthread_attr_t *, int *); -int pthread_attr_setinheritsched(pthread_attr_t *, int); -int pthread_attr_getschedpolicy(const pthread_attr_t *, int *); -int pthread_attr_setschedpolicy(pthread_attr_t *, int); -int pthread_attr_getscope(const pthread_attr_t *, int *); -int pthread_attr_setscope(pthread_attr_t *, int); -int pthread_attr_getstack(const pthread_attr_t *, void **, size_t *); -int pthread_attr_setstack(pthread_attr_t *, void *, size_t); -int pthread_attr_getstacksize(const pthread_attr_t *, size_t *); -int pthread_attr_setstacksize(pthread_attr_t *, size_t); -int pthread_create(pthread_t *, const pthread_attr_t *, void *(*)(void *), - void *); -int pthread_detach(pthread_t); -int pthread_cancel(pthread_t); -int pthread_join(pthread_t, void **); -int pthread_equal(pthread_t, pthread_t); -int pthread_once(pthread_once_t *, void (*)(void)); -int pthread_spin_init(pthread_spinlock_t *, int); -int pthread_spin_destroy(pthread_spinlock_t *); -int pthread_spin_lock(pthread_spinlock_t *); -int pthread_spin_unlock(pthread_spinlock_t *); -int pthread_spin_trylock(pthread_spinlock_t *); -int pthread_mutexattr_init(pthread_mutexattr_t *); -int pthread_mutexattr_destroy(pthread_mutexattr_t *); -int pthread_mutexattr_gettype(const pthread_mutexattr_t *, int *); -int pthread_mutexattr_settype(pthread_mutexattr_t *, int); -int pthread_mutexattr_setpshared(pthread_mutexattr_t *, int); -int pthread_mutexattr_getpshared(const pthread_mutexattr_t *, int *); -int pthread_mutex_init(pthread_mutex_t *, const pthread_mutexattr_t *); -int pthread_mutex_lock(pthread_mutex_t *); -int pthread_mutex_unlock(pthread_mutex_t *); -int pthread_mutex_trylock(pthread_mutex_t *); -int pthread_mutex_destroy(pthread_mutex_t *); -int pthread_mutex_consistent(pthread_mutex_t *); -int pthread_condattr_init(pthread_condattr_t *); -int pthread_condattr_destroy(pthread_condattr_t *); -int pthread_condattr_setpshared(pthread_condattr_t *, int); -int pthread_condattr_getpshared(const pthread_condattr_t *, int *); -int pthread_cond_init(pthread_cond_t *, const pthread_condattr_t *); -int pthread_cond_destroy(pthread_cond_t *); -int pthread_cond_wait(pthread_cond_t *, pthread_mutex_t *); -int pthread_cond_broadcast(pthread_cond_t *); -int pthread_cond_signal(pthread_cond_t *); -int pthread_rwlockattr_init(pthread_rwlockattr_t *); -int pthread_rwlockattr_destroy(pthread_rwlockattr_t *); -int pthread_rwlockattr_setpshared(pthread_rwlockattr_t *, int); -int pthread_rwlockattr_getpshared(const pthread_rwlockattr_t *, int *); -int pthread_rwlock_init(pthread_rwlock_t *, const pthread_rwlockattr_t *); -int pthread_rwlock_destroy(pthread_rwlock_t *); -int pthread_rwlock_rdlock(pthread_rwlock_t *); -int pthread_rwlock_tryrdlock(pthread_rwlock_t *); -int pthread_rwlock_wrlock(pthread_rwlock_t *); -int pthread_rwlock_trywrlock(pthread_rwlock_t *); -int pthread_rwlock_unlock(pthread_rwlock_t *); -int pthread_key_create(pthread_key_t *, pthread_key_dtor); -int pthread_key_delete(pthread_key_t); -int pthread_setspecific(pthread_key_t, void *); -void *pthread_getspecific(pthread_key_t); -int pthread_barrierattr_init(pthread_barrierattr_t *); -int pthread_barrierattr_destroy(pthread_barrierattr_t *); -int pthread_barrierattr_getpshared(const pthread_barrierattr_t *, int *); -int pthread_barrierattr_setpshared(pthread_barrierattr_t *, int); -int pthread_barrier_wait(pthread_barrier_t *); -int pthread_barrier_destroy(pthread_barrier_t *); -int pthread_barrier_init(pthread_barrier_t *, const pthread_barrierattr_t *, - unsigned); - -#define pthread_spin_init(pSpin, multiprocess) ((pSpin)->_lock = 0, 0) -#define pthread_spin_destroy(pSpin) ((pSpin)->_lock = -1, 0) -#if (__GNUC__ + 0) * 100 + (__GNUC_MINOR__ + 0) >= 407 && \ - !defined(__STRICT_ANSI__) -extern const errno_t EBUSY; -#define pthread_spin_lock(pSpin) \ - ({ \ - pthread_spinlock_t *_s = pSpin; \ - while (__atomic_test_and_set(&_s->_lock, __ATOMIC_SEQ_CST)) donothing; \ - 0; \ - }) -#define pthread_spin_unlock(pSpin) \ - ({ \ - pthread_spinlock_t *_s = pSpin; \ - __atomic_store_n(&_s->_lock, 0, __ATOMIC_RELAXED); \ - 0; \ - }) -#define pthread_spin_trylock(pSpin) \ - ({ \ - pthread_spinlock_t *_s = pSpin; \ - __atomic_test_and_set(&_s->_lock, __ATOMIC_SEQ_CST) ? EBUSY : 0; \ - }) -#endif /* GCC 4.7+ */ +void cthread_ungarbage(void); COSMOPOLITAN_C_END_ #endif /* !(__ASSEMBLER__ + __LINKER__ + 0) */ diff --git a/libc/thread/thread.mk b/libc/thread/thread.mk index d32ebd6ce..a953bde03 100644 --- a/libc/thread/thread.mk +++ b/libc/thread/thread.mk @@ -8,8 +8,8 @@ LIBC_THREAD = $(LIBC_THREAD_A_DEPS) $(LIBC_THREAD_A) LIBC_THREAD_A = o/$(MODE)/libc/thread/thread.a LIBC_THREAD_A_FILES := $(wildcard libc/thread/*) LIBC_THREAD_A_HDRS = $(filter %.h,$(LIBC_THREAD_A_FILES)) -LIBC_THREAD_A_SRCS_C = $(filter %.c,$(LIBC_THREAD_A_FILES)) LIBC_THREAD_A_SRCS_S = $(filter %.S,$(LIBC_THREAD_A_FILES)) +LIBC_THREAD_A_SRCS_C = $(filter %.c,$(LIBC_THREAD_A_FILES)) LIBC_THREAD_A_SRCS = \ $(LIBC_THREAD_A_SRCS_S) \ @@ -24,17 +24,15 @@ LIBC_THREAD_A_CHECKS = \ $(LIBC_THREAD_A_HDRS:%=o/$(MODE)/%.ok) LIBC_THREAD_A_DIRECTDEPS = \ + LIBC_STUBS \ LIBC_CALLS \ LIBC_INTRIN \ LIBC_MEM \ - LIBC_NT_KERNEL32 \ LIBC_RUNTIME \ LIBC_STR \ - LIBC_STUBS \ LIBC_SYSV \ LIBC_SYSV_CALLS \ - LIBC_NEXGEN32E \ - THIRD_PARTY_NSYNC + LIBC_NEXGEN32E LIBC_THREAD_A_DEPS := \ $(call uniq,$(foreach x,$(LIBC_THREAD_A_DIRECTDEPS),$($(x)))) @@ -48,10 +46,9 @@ $(LIBC_THREAD_A).pkg: \ $(LIBC_THREAD_A_OBJS) \ $(foreach x,$(LIBC_THREAD_A_DIRECTDEPS),$($(x)_A).pkg) -$(LIBC_THREAD_A_OBJS): private \ - OVERRIDE_CCFLAGS += \ - -ffunction-sections \ - -fdata-sections +o/tinylinux/libc/thread/clone.o: private \ + OVERRIDE_CFLAGS += \ + -ffunction-sections LIBC_THREAD_LIBS = $(foreach x,$(LIBC_THREAD_ARTIFACTS),$($(x))) LIBC_THREAD_SRCS = $(foreach x,$(LIBC_THREAD_ARTIFACTS),$($(x)_SRCS)) diff --git a/libc/thread/tls.h b/libc/thread/tls.h deleted file mode 100644 index f5a3f422d..000000000 --- a/libc/thread/tls.h +++ /dev/null @@ -1,48 +0,0 @@ -#ifndef COSMOPOLITAN_LIBC_THREAD_TLS_H_ -#define COSMOPOLITAN_LIBC_THREAD_TLS_H_ - -#define TLS_ALIGNMENT 64 - -#if !(__ASSEMBLER__ + __LINKER__ + 0) -COSMOPOLITAN_C_START_ - -struct CosmoFtrace { /* 16 */ - bool ft_once; /* 0 */ - bool ft_noreentry; /* 1 */ - int ft_skew; /* 4 */ - int64_t ft_lastaddr; /* 8 */ -}; - -struct CosmoTib { - struct CosmoTib *tib_self; /* 0x00 */ - struct CosmoFtrace tib_ftrace; /* 0x08 */ - void *tib_garbages; /* 0x18 */ - intptr_t tib_locale; /* 0x20 */ - intptr_t tib_pthread; /* 0x28 */ - struct CosmoTib *tib_self2; /* 0x30 */ - int32_t tib_tid; /* 0x38 */ - int32_t tib_errno; /* 0x3c */ -}; - -extern int __threaded; -extern bool __tls_enabled; -extern unsigned __tls_index; - -void __require_tls(void); - -#if defined(__GNUC__) && defined(__x86_64__) && !defined(__STRICT_ANSI__) -/** - * Returns location of thread information block. - * - * This can't be used in privileged functions. - */ -static inline struct CosmoTib *__get_tls(void) { - struct CosmoTib *_tib; - asm("mov\t%%fs:0,%0" : "=r"(_tib) : /* no inputs */ : "memory"); - return _tib; -} -#endif /* GNU x86-64 */ - -COSMOPOLITAN_C_END_ -#endif /* !(__ASSEMBLER__ + __LINKER__ + 0) */ -#endif /* COSMOPOLITAN_LIBC_THREAD_TLS_H_ */ diff --git a/libc/thread/pthread_ungarbage.c b/libc/thread/ungarbage.c similarity index 93% rename from libc/thread/pthread_ungarbage.c rename to libc/thread/ungarbage.c index 0a33e7011..80de172cd 100644 --- a/libc/thread/pthread_ungarbage.c +++ b/libc/thread/ungarbage.c @@ -19,11 +19,12 @@ #include "libc/assert.h" #include "libc/mem/mem.h" #include "libc/nexgen32e/gc.internal.h" -#include "libc/thread/tls.h" +#include "libc/nexgen32e/gettls.h" +#include "libc/thread/thread.h" -void _pthread_ungarbage(void) { +void cthread_ungarbage(void) { struct Garbages *g; - if ((g = __get_tls()->tib_garbages)) { + if ((g = ((cthread_t)__get_tls())->garbages)) { // _pthread_exit() uses _gclongjmp() so if this assertion fails, // then the likely cause is the thread used gc() with longjmp(). assert(!g->i); diff --git a/libc/thread/pthread_zombies.c b/libc/thread/zombie.c similarity index 98% rename from libc/thread/pthread_zombies.c rename to libc/thread/zombie.c index 1fee44614..b93f9d67e 100644 --- a/libc/thread/pthread_zombies.c +++ b/libc/thread/zombie.c @@ -17,11 +17,11 @@ │ PERFORMANCE OF THIS SOFTWARE. │ ╚─────────────────────────────────────────────────────────────────────────────*/ #include "libc/intrin/atomic.h" +#include "libc/intrin/pthread.h" #include "libc/mem/mem.h" #include "libc/runtime/runtime.h" #include "libc/thread/posixthread.internal.h" #include "libc/thread/spawn.h" -#include "libc/thread/thread.h" /** * @fileoverview Memory collector for detached threads. diff --git a/libc/time/localtime.c b/libc/time/localtime.c index f52e4f2f1..407c10c5f 100644 --- a/libc/time/localtime.c +++ b/libc/time/localtime.c @@ -4,10 +4,10 @@ #define LOCALTIME_IMPLEMENTATION #include "libc/calls/calls.h" #include "libc/intrin/bits.h" -#include "libc/intrin/nopl.internal.h" -#include "libc/thread/thread.h" +#include "libc/intrin/nopl.h" +#include "libc/intrin/pthread.h" #include "libc/mem/mem.h" -#include "libc/thread/tls.h" +#include "libc/nexgen32e/threaded.h" #include "libc/runtime/gc.h" #include "libc/str/str.h" #include "libc/sysv/consts/o.h" diff --git a/libc/vga/readv-vga.c b/libc/vga/readv-vga.c deleted file mode 100644 index a9668723f..000000000 --- a/libc/vga/readv-vga.c +++ /dev/null @@ -1,49 +0,0 @@ -/*-*- mode:c;indent-tabs-mode:nil;c-basic-offset:2;tab-width:8;coding:utf-8 -*-│ -│vi: set net ft=c ts=2 sts=2 sw=2 fenc=utf-8 :vi│ -╞══════════════════════════════════════════════════════════════════════════════╡ -│ This is free and unencumbered software released into the public domain. │ -│ │ -│ Anyone is free to copy, modify, publish, use, compile, sell, or │ -│ distribute this software, either in source code form or as a compiled │ -│ binary, for any purpose, commercial or non-commercial, and by any │ -│ means. │ -│ │ -│ In jurisdictions that recognize copyright laws, the author or authors │ -│ of this software dedicate any and all copyright interest in the │ -│ software to the public domain. We make this dedication for the benefit │ -│ of the public at large and to the detriment of our heirs and │ -│ successors. We intend this dedication to be an overt act of │ -│ relinquishment in perpetuity of all present and future rights to this │ -│ software under copyright law. │ -│ │ -│ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, │ -│ EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF │ -│ MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. │ -│ IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR │ -│ OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, │ -│ ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR │ -│ OTHER DEALINGS IN THE SOFTWARE. │ -╚─────────────────────────────────────────────────────────────────────────────*/ -#include "libc/calls/struct/fd.internal.h" -#include "libc/calls/struct/iovec.h" -#include "libc/calls/struct/iovec.internal.h" -#include "libc/vga/vga.internal.h" - -ssize_t sys_readv_vga(struct Fd *fd, const struct iovec *iov, int iovlen) { - /* NOTE: this routine is always non-blocking. */ - size_t i, redd = 0; - ssize_t res = 0; - for (i = 0; i < iovlen; ++i) { - void *input = iov[i].iov_base; - size_t len = iov[i].iov_len; - res = _TtyRead(&_vga_tty, input, len); - if (res < 0) - break; - redd += res; - if (redd != len) - return redd; - } - if (!redd) - return res; - return redd; -} diff --git a/libc/vga/tty.c b/libc/vga/tty.c index b76129a55..76d6c7f96 100644 --- a/libc/vga/tty.c +++ b/libc/vga/tty.c @@ -93,7 +93,7 @@ static wchar_t *Wcs(struct Tty *tty) void _StartTty(struct Tty *tty, unsigned short yn, unsigned short xn, unsigned short starty, unsigned short startx, - unsigned char chr_ht, void *vccs, wchar_t *wcs) { + void *vccs, wchar_t *wcs) { struct VgaTextCharCell *ccs = vccs; memset(tty, 0, sizeof(struct Tty)); SetYn(tty, yn); @@ -103,11 +103,8 @@ void _StartTty(struct Tty *tty, unsigned short yn, unsigned short xn, starty = yn - 1; if (startx >= xn) startx = xn - 1; - if (chr_ht > 32) - chr_ht = 32; tty->y = starty; tty->x = startx; - tty->chr_ht = chr_ht; if (SetWcs(tty, wcs)) { size_t n = (size_t)yn * xn, i; for (i = 0; i < n; ++i) @@ -146,27 +143,66 @@ static wchar_t *GetXlatLineDrawing(void) { return xlat; } +static void XlatAlphabet(wchar_t xlat[128], int a, int b) { + unsigned i; + for (i = 0; i < 128; ++i) { + if ('a' <= i && i <= 'z') { + xlat[i] = i - 'a' + a; + } else if ('A' <= i && i <= 'Z') { + xlat[i] = i - 'A' + b; + } else { + xlat[i] = i; + } + } +} + static wchar_t *GetXlatItalic(void) { - /* Unimplemented. Simply output normal non-italicized characters for now. */ - return GetXlatAscii(); + static bool once; + static wchar_t xlat[128]; + if (!once) { + XlatAlphabet(xlat, L'𝑎', L'𝐴'); + once = true; + } + return xlat; } static wchar_t *GetXlatBoldItalic(void) { - /* - * Unimplemented. Simply output high-intensity non-italicized characters - * for now. - */ - return GetXlatAscii(); + static bool once; + static wchar_t xlat[128]; + if (!once) { + XlatAlphabet(xlat, L'𝒂', L'𝑨'); + once = true; + } + return xlat; } static wchar_t *GetXlatBoldFraktur(void) { - /* Unimplemented. */ - return GetXlatAscii(); + static bool once; + static wchar_t xlat[128]; + if (!once) { + XlatAlphabet(xlat, L'𝖆', L'𝕬'); + once = true; + } + return xlat; } static wchar_t *GetXlatFraktur(void) { - /* Unimplemented. */ - return GetXlatAscii(); + unsigned i; + static bool once; + static wchar_t xlat[128]; + if (!once) { + for (i = 0; i < ARRAYLEN(xlat); ++i) { + if ('A' <= i && i <= 'Z') { + xlat[i] = L"𝔄𝔅ℭ𝔇𝔈𝔉𝔊ℌℑ𝔍𝔎𝔏𝔐𝔑𝔒𝔓𝔔ℜ𝔖𝔗𝔘𝔙𝔚𝔛𝔜ℨ"[i - 'A']; + } else if ('a' <= i && i <= 'z') { + xlat[i] = i - 'a' + L'𝔞'; + } else { + xlat[i] = i; + } + } + once = true; + } + return xlat; } static wchar_t *GetXlatDoubleWidth(void) { @@ -222,33 +258,15 @@ static void TtySetCodepage(struct Tty *tty, char id) { } } -/** - * Map the currently active foreground & background colors & terminal - * configuration to a VGA text character attribute byte. - * - * @see VGA_USE_BLINK macro (libc/vga/vga.internal.h) - * @see drivers/tty/vt/vt.c in Linux 5.9.14 source code - */ static uint8_t TtyGetVgaAttr(struct Tty *tty) { - uint8_t attr; - if ((tty->pr & kTtyFlip) == 0) - attr = tty->fg | tty->bg << 4; - else - attr = tty->bg | tty->fg << 4; + uint8_t attr = tty->fg | tty->bg << 4; + if ((tty->pr & kTtyBold) != 0) + attr |= 0x08; #ifdef VGA_USE_BLINK - /* - * If blinking is enabled, we can only have the 8 dark background color - * codes (0 to 7). Simply map any bright background color (8 to 15) to - * its corresponding dark color, & call it a day. This is a bit more - * simplistic than what Linux does, but should be enough. - */ - attr &= ~0x80; if ((tty->pr & kTtyBlink) != 0) attr |= 0x80; #endif - if ((tty->pr & kTtyBold) != 0) - attr |= 0x08; return attr; } @@ -553,18 +571,15 @@ static void TtyDeleteLines(struct Tty *tty) { } static void TtyReportDeviceStatus(struct Tty *tty) { - static const char report[] = "\e[0n"; - _TtyWriteInput(tty, report, strlen(report)); + _TtyWriteInput(tty, "\e[0n", 4); } static void TtyReportPreferredVtType(struct Tty *tty) { - static const char report[] = "\e[?1;0c"; - _TtyWriteInput(tty, report, strlen(report)); + _TtyWriteInput(tty, "\e[?1;0c", 4); } static void TtyReportPreferredVtIdentity(struct Tty *tty) { - static const char report[] = "\e/Z"; - _TtyWriteInput(tty, report, strlen(report)); + _TtyWriteInput(tty, "\e/Z", 4); } static void TtyBell(struct Tty *tty) { @@ -624,23 +639,37 @@ static void TtyCsiN(struct Tty *tty) { /** * Map the given (R, G, B) triplet to one of the 16 basic foreground colors - * or one of the 16 background colors. + * or one of the 8 (or 16) background colors. * + * @see VGA_USE_BLINK macro (libc/vga/vga.internal.h) * @see drivers/tty/vt/vt.c in Linux 5.9.14 source code */ -static uint8_t TtyMapTrueColor(uint8_t r, uint8_t g, uint8_t b) +static uint8_t TtyMapTrueColor(uint8_t r, uint8_t g, uint8_t b, bool as_fg) { - uint8_t hue = 0, max = MAX(MAX(r, g), b); - if (r > max / 2) - hue |= 4; - if (g > max / 2) - hue |= 2; - if (b > max / 2) - hue |= 1; - if (hue == 7 && max <= 0x55) - hue = 8; - else if (max > 0xaa) - hue |= 8; + uint8_t hue = 0; +#ifdef VGA_USE_BLINK + if (!as_fg) { + if (r >= 128) + hue |= 4; + if (g >= 128) + hue |= 2; + if (b >= 128) + hue |= 1; + } else +#endif + { + uint8_t max = MAX(MAX(r, g), b); + if (r > max / 2) + hue |= 4; + if (g > max / 2) + hue |= 2; + if (b > max / 2) + hue |= 1; + if (hue == 7 && max <= 0x55) + hue = 8; + else if (max > 0xaa) + hue |= 8; + } return hue; } @@ -650,7 +679,7 @@ static uint8_t TtyMapTrueColor(uint8_t r, uint8_t g, uint8_t b) * * @see drivers/tty/vt/vt.c in Linux 5.9.14 source code */ -static uint8_t TtyMapXtermColor(uint8_t color) +static uint8_t TtyMapXtermColor(uint8_t color, bool as_fg) { uint8_t r, g, b; if (color < 8) { @@ -670,7 +699,7 @@ static uint8_t TtyMapXtermColor(uint8_t color) r = color * 0x55 / 2; } else r = g = b = (unsigned)color * 10 - 2312; - return TtyMapTrueColor(r, g, b); + return TtyMapTrueColor(r, g, b, as_fg); } static void TtySelectGraphicsRendition(struct Tty *tty) { @@ -793,7 +822,13 @@ static void TtySelectGraphicsRendition(struct Tty *tty) { break; case 100 ... 107: code[0] -= 100 - 40; +#ifndef VGA_USE_BLINK + /* + * If blinking is not enabled in VGA text mode, then we can + * use bright background colors. + */ code[0] += 8; +#endif /* fallthrough */ case 40 ... 47: tty->bg = code[0] - 40; @@ -820,7 +855,7 @@ static void TtySelectGraphicsRendition(struct Tty *tty) { if (++code[3] == 3) { code[3] = 0; t = kSgr; - tty->fg = TtyMapTrueColor(code[0], code[1], code[2]); + tty->fg = TtyMapTrueColor(code[0], code[1], code[2], true); tty->pr |= kTtyFg; tty->pr |= kTtyTrue; } @@ -829,20 +864,20 @@ static void TtySelectGraphicsRendition(struct Tty *tty) { if (++code[3] == 3) { code[3] = 0; t = kSgr; - tty->bg = TtyMapTrueColor(code[0], code[1], code[2]); + tty->bg = TtyMapTrueColor(code[0], code[1], code[2], false); tty->pr |= kTtyBg; tty->pr |= kTtyTrue; } break; case kSgrFgXterm: t = kSgr; - tty->fg = TtyMapXtermColor(code[0]); + tty->fg = TtyMapXtermColor(code[0], true); tty->pr |= kTtyFg; tty->pr &= ~kTtyTrue; break; case kSgrBgXterm: t = kSgr; - tty->bg = TtyMapXtermColor(code[0]); + tty->bg = TtyMapXtermColor(code[0], false); tty->pr |= kTtyBg; tty->pr &= ~kTtyTrue; break; @@ -1046,17 +1081,10 @@ static void TtyEscAppend(struct Tty *tty, char c) { } static void TtyUpdateHwCursor(struct Tty *tty) { - unsigned char start = tty->chr_ht - 2, end = tty->chr_ht - 1; unsigned short pos = tty->y * Xn(tty) + tty->x; - if ((tty->conf & kTtyNocursor)) - start |= 1 << 5; - outb(CRTPORT, 0x0A); - outb(CRTPORT + 1, start); - outb(CRTPORT, 0x0B); - outb(CRTPORT + 1, end); - outb(CRTPORT, 0x0E); + outb(CRTPORT, 0x0e); outb(CRTPORT + 1, (unsigned char)(pos >> 8)); - outb(CRTPORT, 0x0F); + outb(CRTPORT, 0x0f); outb(CRTPORT + 1, (unsigned char)pos); } @@ -1176,11 +1204,9 @@ ssize_t _TtyWriteInput(struct Tty *tty, const void *data, size_t n) { if (cr && i < sizeof tty->input.p) { p[i++] = '\n'; } -#ifdef VGA_PERSNICKETY_STATUS if (!(tty->conf & kTtyNoecho)) { _TtyWrite(tty, p + tty->input.i, i - tty->input.i); } -#endif tty->input.i = i; return n; } @@ -1189,7 +1215,6 @@ ssize_t _TtyRead(struct Tty *tty, void *buf, size_t size) { char *p; size_t n; n = MIN(size, tty->input.i); -#ifdef VGA_PERSNICKETY_STATUS if (!(tty->conf & kTtyNocanon)) { if ((p = memchr(tty->input.p, '\n', n))) { n = MIN(n, tty->input.p - p + 1); @@ -1197,7 +1222,6 @@ ssize_t _TtyRead(struct Tty *tty, void *buf, size_t size) { n = 0; } } -#endif memcpy(buf, tty->input.p, n); memcpy(tty->input.p, tty->input.p + n, tty->input.i - n); tty->input.i -= n; diff --git a/libc/vga/vga-init.S b/libc/vga/vga-init.S index 7e332a5d7..8b4c657cc 100644 --- a/libc/vga/vga-init.S +++ b/libc/vga/vga-init.S @@ -72,4 +72,3 @@ vga_console: .endobj vga_console,globl,hidden .previous .yoink sys_writev_vga - .yoink sys_readv_vga diff --git a/libc/vga/vga.internal.h b/libc/vga/vga.internal.h index bfc6488e1..134303139 100644 --- a/libc/vga/vga.internal.h +++ b/libc/vga/vga.internal.h @@ -60,14 +60,6 @@ * (https://www.delorie.com/djgpp/doc/rbinter/id/22/1.html) */ #undef VGA_USE_BLINK -/** - * If VGA_PERSNICKETY_STATUS is defined, then when deciding how to return - * status response codes (e.g. "\e[0n"), the tty code will pay attention to - * the terminal's termios mode (TODO). If undefined, the tty code will - * simply return any response strings immediately, & will not echo them — - * per the common use case. - */ -#undef VGA_PERSNICKETY_STATUS #define kTtyFg 0x0001 #define kTtyBg 0x0002 @@ -117,7 +109,7 @@ struct Tty { uint32_t u8; uint32_t n8; uint32_t pr; - uint8_t fg, bg, chr_ht; + uint8_t fg, bg; uint32_t conf; unsigned short savey, savex; struct VgaTextCharCell *ccs; @@ -142,8 +134,7 @@ struct Tty { }; void _StartTty(struct Tty *, unsigned short, unsigned short, - unsigned short, unsigned short, unsigned char, - void *, wchar_t *); + unsigned short, unsigned short, void *, wchar_t *); ssize_t _TtyRead(struct Tty *, void *, size_t); ssize_t _TtyWrite(struct Tty *, const void *, size_t); ssize_t _TtyWriteInput(struct Tty *, const void *, size_t); @@ -154,9 +145,6 @@ void _TtyErase(struct Tty *, size_t, size_t); void _TtySetY(struct Tty *, unsigned short); void _TtySetX(struct Tty *, unsigned short); -extern struct Tty _vga_tty; - -ssize_t sys_readv_vga(struct Fd *, const struct iovec *, int); ssize_t sys_writev_vga(struct Fd *, const struct iovec *, int); COSMOPOLITAN_C_END_ diff --git a/libc/vga/writev-vga.c b/libc/vga/writev-vga.c index d7a7afa1d..854596441 100644 --- a/libc/vga/writev-vga.c +++ b/libc/vga/writev-vga.c @@ -31,21 +31,20 @@ #include "libc/runtime/pc.internal.h" #include "libc/str/str.h" +static struct Tty vga_tty; #ifdef VGA_USE_WCS static wchar_t vga_wcs[VGA_TTY_HEIGHT * VGA_TTY_WIDTH]; #else static wchar_t * const vga_wcs = NULL; #endif -struct Tty _vga_tty; - ssize_t sys_writev_vga(struct Fd *fd, const struct iovec *iov, int iovlen) { size_t i, wrote = 0; ssize_t res = 0; for (i = 0; i < iovlen; ++i) { - void *output = iov[i].iov_base; + void *input = iov[i].iov_base; size_t len = iov[i].iov_len; - res = _TtyWrite(&_vga_tty, output, len); + res = _TtyWrite(&vga_tty, input, len); if (res < 0) break; wrote += res; @@ -59,23 +58,15 @@ ssize_t sys_writev_vga(struct Fd *fd, const struct iovec *iov, int iovlen) { __attribute__((__constructor__)) static textstartup void _vga_init(void) { void * const vid_buf = (void *)(BANE + 0xb8000ull); - /* - * Get the initial cursor position from the BIOS data area. Also get - * the height (in scan lines) of each character; this is used to set the - * cursor shape. - */ + /* Get the initial cursor position from the BIOS data area. */ typedef struct { unsigned char col, row; } bios_curs_pos_t; bios_curs_pos_t pos = *(bios_curs_pos_t *)(BANE + 0x0450ull); - uint8_t chr_ht = *(uint8_t *)(BANE + 0x0485ull), - chr_ht_hi = *(uint8_t *)(BANE + 0x0486ull); - if (chr_ht_hi != 0 || chr_ht > 32) - chr_ht = 32; /* - * Initialize our tty structure from the current screen contents, current - * cursor position, & character height. + * Initialize our tty structure from the current screen contents & current + * cursor position. */ - _StartTty(&_vga_tty, VGA_TTY_HEIGHT, VGA_TTY_WIDTH, pos.row, pos.col, - chr_ht, vid_buf, vga_wcs); + _StartTty(&vga_tty, VGA_TTY_HEIGHT, VGA_TTY_WIDTH, pos.row, pos.col, + vid_buf, vga_wcs); } diff --git a/libc/zipos/free.c b/libc/zipos/free.c index 512a71eba..5962dcc3a 100644 --- a/libc/zipos/free.c +++ b/libc/zipos/free.c @@ -23,6 +23,11 @@ #include "libc/str/str.h" #include "libc/zipos/zipos.internal.h" +/** + * Frees ZipOS handle. + * @asyncsignalsafe + * @threadsafe + */ void __zipos_free(struct Zipos *z, struct ZiposHandle *h) { if (IsAsan()) { __asan_poison((char *)h + sizeof(struct ZiposHandle), diff --git a/libc/zipos/get.c b/libc/zipos/get.c index 352b3475d..6511d5526 100644 --- a/libc/zipos/get.c +++ b/libc/zipos/get.c @@ -20,7 +20,7 @@ #include "libc/calls/strace.internal.h" #include "libc/intrin/cmpxchg.h" #include "libc/intrin/promises.internal.h" -#include "libc/thread/thread.h" +#include "libc/intrin/pthread.h" #include "libc/macros.internal.h" #include "libc/runtime/runtime.h" #include "libc/sysv/consts/map.h" diff --git a/libc/zipos/lock.c b/libc/zipos/lock.c index 464619308..1b86a9346 100644 --- a/libc/zipos/lock.c +++ b/libc/zipos/lock.c @@ -16,7 +16,7 @@ │ TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR │ │ PERFORMANCE OF THIS SOFTWARE. │ ╚─────────────────────────────────────────────────────────────────────────────*/ -#include "libc/thread/thread.h" +#include "libc/intrin/pthread.h" #include "libc/zipos/zipos.internal.h" static pthread_mutex_t __zipos_lock_obj; diff --git a/libc/zipos/open.c b/libc/zipos/open.c index 44979f08e..ad6ad9918 100644 --- a/libc/zipos/open.c +++ b/libc/zipos/open.c @@ -18,7 +18,6 @@ ╚─────────────────────────────────────────────────────────────────────────────*/ #include "libc/assert.h" #include "libc/calls/calls.h" -#include "libc/calls/extend.internal.h" #include "libc/calls/internal.h" #include "libc/calls/state.internal.h" #include "libc/calls/struct/sigset.h" @@ -43,19 +42,47 @@ #include "libc/zip.h" #include "libc/zipos/zipos.internal.h" -static char *mapend; -static size_t maptotal; +static volatile size_t maptotal; + +static pureconst size_t __zipos_granularity(void) { + return FRAMESIZE * (IsAsan() ? 8 : 1); +} static void *__zipos_mmap(size_t mapsize) { - char *start; size_t offset; + struct DirectMap dm; + int rc, prot, flags; + uint64_t addr, addr2, addr3; assert(mapsize); - offset = maptotal; - maptotal += mapsize; - start = (char *)0x6fd000040000; - if (!mapend) mapend = start; - mapend = _extend(start, maptotal, mapend, 0x6fdfffff0000); - return start + offset; + do offset = maptotal; + while (!_cmpxchg(&maptotal, offset, maptotal + mapsize)); + if (offset + mapsize <= kMemtrackZiposSize) { + prot = PROT_READ | PROT_WRITE; + flags = MAP_PRIVATE | MAP_ANONYMOUS | MAP_FIXED; + addr = kMemtrackZiposStart + offset; + if ((dm = sys_mmap((void *)addr, mapsize, prot, flags, -1, 0)).addr != + MAP_FAILED) { + rc = TrackMemoryInterval(&_mmi, addr >> 16, (addr + mapsize - 1) >> 16, + dm.maphandle, prot, flags, false, false, 0, + mapsize); + assert(!rc); + if (IsAsan()) { + addr2 = (addr >> 3) + 0x7fff8000; + addr3 = ((addr + mapsize) >> 3) + 0x7fff8000; + assert(!(addr2 & (FRAMESIZE - 1))); + assert(!(addr3 & (FRAMESIZE - 1))); + dm = sys_mmap((void *)addr2, mapsize >> 3, prot, flags, -1, 0); + assert(dm.addr != MAP_FAILED); + rc = TrackMemoryInterval(&_mmi, addr2 >> 16, (addr3 >> 16) - 1, + dm.maphandle, prot, flags, false, false, 0, + mapsize >> 3); + assert(!rc); + } + return (void *)addr; + } + } + enomem(); + return 0; } static struct ZiposHandle *__zipos_alloc(struct Zipos *zipos, size_t size) { @@ -63,7 +90,7 @@ static struct ZiposHandle *__zipos_alloc(struct Zipos *zipos, size_t size) { struct ZiposHandle *h, **ph; __zipos_lock(); mapsize = sizeof(struct ZiposHandle) + size; - mapsize = ROUNDUP(mapsize, 4096); + mapsize = ROUNDUP(mapsize, __zipos_granularity()); StartOver: ph = &zipos->freelist; while ((h = *ph)) { diff --git a/libc/zipos/zipos.h b/libc/zipos/zipos.h deleted file mode 100755 index e69de29bb..000000000 diff --git a/libc/zipos/zipos.internal.h b/libc/zipos/zipos.internal.h index 764916634..6fb612e6a 100644 --- a/libc/zipos/zipos.internal.h +++ b/libc/zipos/zipos.internal.h @@ -1,7 +1,8 @@ #ifndef COSMOPOLITAN_LIBC_ZIPOS_ZIPOS_H_ #define COSMOPOLITAN_LIBC_ZIPOS_ZIPOS_H_ -#include "libc/intrin/nopl.internal.h" -#include "libc/thread/tls.h" +#include "libc/calls/calls.h" +#include "libc/intrin/nopl.h" +#include "libc/nexgen32e/threaded.h" #if !(__ASSEMBLER__ + __LINKER__ + 0) COSMOPOLITAN_C_START_ diff --git a/test/libc/calls/fcntl_test.c b/test/libc/calls/fcntl_test.c index dee11c127..e4c4a4ce6 100644 --- a/test/libc/calls/fcntl_test.c +++ b/test/libc/calls/fcntl_test.c @@ -73,6 +73,58 @@ TEST(fcntl, getfd) { void OnSig(int sig) { } +TEST(posixAdvisoryLocks, oneProcess_unlockedFromOwnPerspectiveHuh) { + struct flock lock; + ASSERT_SYS(0, 3, open("foo", O_RDWR | O_CREAT | O_TRUNC, 0644)); + ASSERT_SYS(0, 5, write(3, "hello", 5)); + + // set lock + lock.l_type = F_WRLCK; + lock.l_whence = SEEK_SET; + lock.l_start = 1; + lock.l_len = 3; + lock.l_pid = -2; + ASSERT_SYS(0, 0, fcntl(3, F_SETLK, &lock)); + EXPECT_EQ(F_WRLCK, lock.l_type); + EXPECT_EQ(SEEK_SET, lock.l_whence); + EXPECT_EQ(1, lock.l_start); + EXPECT_EQ(3, lock.l_len); + EXPECT_EQ(-2, lock.l_pid); + + ASSERT_SYS(0, 4, open("foo", O_RDWR)); + + // try lock + lock.l_type = F_WRLCK; + lock.l_whence = SEEK_SET; + lock.l_start = 0; + lock.l_len = 0; + lock.l_pid = -1; + ASSERT_SYS(0, 0, fcntl(4, F_SETLK, &lock)); + EXPECT_EQ(F_WRLCK, lock.l_type); + EXPECT_EQ(SEEK_SET, lock.l_whence); + EXPECT_EQ(0, lock.l_start); + EXPECT_EQ(0, lock.l_len); + EXPECT_EQ(-1, lock.l_pid); + + // get lock information + if (!IsWindows()) { + lock.l_type = F_RDLCK; + lock.l_whence = SEEK_SET; + lock.l_start = 0; + lock.l_len = 0; + lock.l_pid = -7; + ASSERT_SYS(0, 0, fcntl(4, F_GETLK, &lock)); + EXPECT_EQ(F_UNLCK, lock.l_type); + EXPECT_EQ(SEEK_SET, lock.l_whence); + EXPECT_EQ(0, lock.l_start); + EXPECT_EQ(0, lock.l_len); + EXPECT_EQ(-7, lock.l_pid); // doesn't change due to F_UNLCK + } + + ASSERT_SYS(0, 0, close(4)); + ASSERT_SYS(0, 0, close(3)); +} + TEST(posixAdvisoryLocks, twoProcesses) { if (IsWindows()) return; // due to signals diff --git a/test/libc/calls/lock_test.c b/test/libc/calls/lock_test.c deleted file mode 100644 index 7858733b0..000000000 --- a/test/libc/calls/lock_test.c +++ /dev/null @@ -1,183 +0,0 @@ -/*-*- mode:c;indent-tabs-mode:nil;c-basic-offset:2;tab-width:8;coding:utf-8 -*-│ -│vi: set net ft=c ts=2 sts=2 sw=2 fenc=utf-8 :vi│ -╞══════════════════════════════════════════════════════════════════════════════╡ -│ Copyright 2022 Justine Alexandra Roberts Tunney │ -│ │ -│ Permission to use, copy, modify, and/or distribute this software for │ -│ any purpose with or without fee is hereby granted, provided that the │ -│ above copyright notice and this permission notice appear in all copies. │ -│ │ -│ THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL │ -│ WARRANTIES WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED │ -│ WARRANTIES OF MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE │ -│ AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL │ -│ DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR │ -│ PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER │ -│ TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR │ -│ PERFORMANCE OF THIS SOFTWARE. │ -╚─────────────────────────────────────────────────────────────────────────────*/ -#include "libc/calls/calls.h" -#include "libc/calls/struct/flock.h" -#include "libc/dce.h" -#include "libc/errno.h" -#include "libc/sysv/consts/f.h" -#include "libc/sysv/consts/o.h" -#include "libc/testlib/testlib.h" -#include "libc/thread/thread.h" - -/** - * @fileoverview POSIX Advisory Locks Test - */ - -#define PROCESSES 8 -#define THREADS (IsWindows() ? 8 : 0) -#define RATIO 3 -#define ITERATIONS 10 - -char testlib_enable_tmp_setup_teardown; - -_Thread_local const char *kind; - -void Log(const char *fmt, ...) { -#if 0 - va_list va; - char b[512]; - int i, n = sizeof(b); - va_start(va, fmt); - i = ksnprintf(b, n, "%s pid=%d tid=%d ", kind, getpid(), gettid()); - i += kvsnprintf(b + i, MAX(0, n - i), fmt, va); - i += ksnprintf(b + i, MAX(0, n - i), "\n"); - write(2, b, MIN(i, n)); - va_end(va); -#endif -} - -void Lock(int fd, int type, long start, long len) { - int e; - struct flock lock = { - .l_type = type, - .l_whence = SEEK_SET, - .l_start = start, - .l_len = len, - }; - e = errno; - while (fcntl(fd, F_SETLK, &lock)) { - ASSERT_TRUE(errno == EAGAIN || errno == EACCES); - errno = e; - Log("flock spinning on %d", fd); - } -} - -void WriteLock(int fd, long start, long len) { - Lock(fd, F_WRLCK, start, len); - Log("acquired write lock on %d", fd); -} - -void ReadLock(int fd, long start, long len) { - Lock(fd, F_RDLCK, start, len); - Log("acquired read lock on %d", fd); -} - -void Unlock(int fd, long start, long len) { - Lock(fd, F_UNLCK, start, len); - Log("released lock on %d", fd); -} - -void *Reader(void *arg) { - int i, j, fd; - char buf[10]; - kind = arg; - ASSERT_NE(-1, (fd = open("db", O_RDONLY))); - for (j = 0; j < ITERATIONS; ++j) { - ReadLock(fd, 10, 10); - for (i = 0; i < 10; ++i) { - ASSERT_SYS(0, 1, pread(fd, buf + i, 1, 10 + i)); - ASSERT_EQ(buf[0], buf[i]); - } - Unlock(fd, 10, 10); - sched_yield(); - } - ASSERT_SYS(0, 0, close(fd)); - return 0; -} - -void *Writer(void *arg) { - int i, j, fd; - char buf[10]; - kind = arg; - ASSERT_NE(-1, (fd = open("db", O_RDWR))); - for (j = 0; j < ITERATIONS; ++j) { - WriteLock(fd, 10, 10); - for (i = 0; i < 10; ++i) { - ASSERT_SYS(0, 1, pread(fd, buf + i, 1, 10 + i)); - ASSERT_EQ(buf[0], buf[i]); - } - for (i = 0; i < 10; ++i) { - buf[i]++; - } - for (i = 0; i < 10; ++i) { - ASSERT_SYS(0, 1, pwrite(fd, buf + i, 1, 10 + i)); - } - Unlock(fd, 10, 10); - sched_yield(); - } - ASSERT_SYS(0, 0, close(fd)); - return 0; -} - -TEST(posixAdvisoryLocks, threadsAndProcessesFightingForLock) { - int i, rc, pid, fd, ws; - pthread_t th[THREADS + 1]; - - ASSERT_SYS(0, 3, creat("db", 0644)); - ASSERT_SYS(0, 0, ftruncate(3, 30)); - ASSERT_SYS(0, 0, close(3)); - - for (i = 0; i < THREADS; ++i) { - if (i % RATIO == 0) { - ASSERT_EQ(0, pthread_create(th + i, 0, Reader, "reader thread")); - } else { - ASSERT_EQ(0, pthread_create(th + i, 0, Writer, "writer thread")); - } - } - - for (i = 0; i < PROCESSES; ++i) { - ASSERT_NE(-1, (rc = fork())); - if (!rc) { - if (i % RATIO == 0) { - Writer("writer process"); - } else { - Reader("reader process"); - } - _Exit(0); - } - } - - ASSERT_NE(-1, (fd = open("db", O_RDWR))); - Lock(fd, F_WRLCK, 0, 10); - Lock(fd, F_WRLCK, 20, 10); - - for (i = 0; i < THREADS; ++i) { - ASSERT_EQ(0, pthread_join(th[i], 0)); - } - - kind = "main process"; - for (;;) { - int e = errno; - if ((pid = waitpid(0, &ws, 0)) != -1) { - if (WIFSIGNALED(ws)) { - Log("process %d terminated with %G", pid, WTERMSIG(ws)); - testlib_incrementfailed(); - } else if (WEXITSTATUS(ws)) { - Log("process %d exited with %d", pid, WEXITSTATUS(ws)); - testlib_incrementfailed(); - } - } else { - ASSERT_EQ(ECHILD, errno); - errno = e; - break; - } - } - - ASSERT_SYS(0, 0, close(fd)); -} diff --git a/test/libc/calls/pledge_test.c b/test/libc/calls/pledge_test.c index f777df43d..b037d50f8 100644 --- a/test/libc/calls/pledge_test.c +++ b/test/libc/calls/pledge_test.c @@ -36,7 +36,7 @@ #include "libc/runtime/runtime.h" #include "libc/sock/sock.h" #include "libc/sock/struct/sockaddr.h" -#include "libc/stdio/lock.internal.h" +#include "libc/stdio/lock.h" #include "libc/stdio/stdio.h" #include "libc/sysv/consts/af.h" #include "libc/sysv/consts/at.h" @@ -560,7 +560,7 @@ TEST(pledge_openbsd, bigSyscalls) { int LockWorker(void *arg, int tid) { flockfile(stdout); - ASSERT_EQ(gettid(), stdout->lock._lock & 0x000fffff); + ASSERT_EQ(gettid(), stdout->lock.lock); funlockfile(stdout); return 0; } diff --git a/test/libc/calls/preadv_test.c b/test/libc/calls/preadv_test.c deleted file mode 100644 index 334918e93..000000000 --- a/test/libc/calls/preadv_test.c +++ /dev/null @@ -1,88 +0,0 @@ -/*-*- mode:c;indent-tabs-mode:nil;c-basic-offset:2;tab-width:8;coding:utf-8 -*-│ -│vi: set net ft=c ts=2 sts=2 sw=2 fenc=utf-8 :vi│ -╞══════════════════════════════════════════════════════════════════════════════╡ -│ Copyright 2022 Justine Alexandra Roberts Tunney │ -│ │ -│ Permission to use, copy, modify, and/or distribute this software for │ -│ any purpose with or without fee is hereby granted, provided that the │ -│ above copyright notice and this permission notice appear in all copies. │ -│ │ -│ THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL │ -│ WARRANTIES WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED │ -│ WARRANTIES OF MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE │ -│ AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL │ -│ DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR │ -│ PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER │ -│ TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR │ -│ PERFORMANCE OF THIS SOFTWARE. │ -╚─────────────────────────────────────────────────────────────────────────────*/ -#include "libc/calls/calls.h" -#include "libc/calls/struct/iovec.h" -#include "libc/errno.h" -#include "libc/sysv/consts/o.h" -#include "libc/testlib/testlib.h" - -char testlib_enable_tmp_setup_teardown; - -TEST(preadv, ebadf) { - EXPECT_SYS(EBADF, -1, preadv(-1, 0, 0, 0)); - EXPECT_SYS(EBADF, -1, pwritev(-1, 0, 0, 0)); -} - -TEST(preadv, testOneIovecWorks_itDelegatesToPread) { - char b1[2] = "hi"; - struct iovec v[1] = {{b1, 2}}; - ASSERT_SYS(0, 3, open("foo", O_RDWR | O_CREAT | O_TRUNC, 0644)); - EXPECT_SYS(0, 2, pwritev(3, v, 1, 0)); - b1[0] = 0; - b1[1] = 0; - EXPECT_SYS(0, 0, preadv(3, 0, 0, 0)); - EXPECT_SYS(0, 0, pwritev(3, 0, 0, 0)); - EXPECT_SYS(0, 2, preadv(3, v, 1, 0)); - ASSERT_EQ('h', b1[0]); - ASSERT_EQ('i', b1[1]); - b1[0] = 0; - b1[1] = 0x55; - EXPECT_SYS(0, 1, preadv(3, v, 1, 1)); - ASSERT_EQ('i', b1[0]); - ASSERT_EQ(0x55, b1[1]); - ASSERT_SYS(0, 0, close(3)); -} - -TEST(preadv, testTwoIovecWorks_itMustDoItsThing) { - char b1[1] = "h"; - char b2[1] = "i"; - struct iovec v[2] = {{b1, 1}, {b2, 1}}; - ASSERT_SYS(0, 3, open("foo", O_RDWR | O_CREAT | O_TRUNC, 0644)); - EXPECT_SYS(0, 2, pwritev(3, v, 2, 0)); - b1[0] = 0; - b2[0] = 0; - EXPECT_SYS(0, 2, preadv(3, v, 2, 0)); - ASSERT_EQ('h', b1[0]); - ASSERT_EQ('i', b2[0]); - b1[0] = 0; - b2[0] = 0x55; - EXPECT_SYS(0, 1, preadv(3, v, 2, 1)); - ASSERT_EQ('i', b1[0]); - ASSERT_EQ(0x55, b2[0]); - ASSERT_SYS(0, 0, close(3)); -} - -TEST(pwritev, writePastEof_zeroExtendsFile_alsoDoesntChangeFilePointer) { - char b[6] = {0x55, 0x55, 0x55, 0x55, 0x55, 0x55}; - char s[2] = "hi"; - struct iovec v[1] = {{s, 2}}; - ASSERT_SYS(0, 3, open("foo", O_RDWR | O_CREAT | O_TRUNC, 0644)); - ASSERT_SYS(EINVAL, -1, pwritev(3, v, 1, -999)); - ASSERT_SYS(0, 2, pwritev(3, v, 1, 0)); - ASSERT_SYS(0, 2, pwritev(3, v, 1, 4)); - ASSERT_SYS(0, 2, preadv(3, v, 1, 0)); - ASSERT_SYS(0, 6, read(3, b, 6)); - ASSERT_EQ('h', b[0]); - ASSERT_EQ('i', b[1]); - ASSERT_EQ(0, b[2]); - ASSERT_EQ(0, b[3]); - ASSERT_EQ('h', b[4]); - ASSERT_EQ('i', b[5]); - ASSERT_SYS(0, 0, close(3)); -} diff --git a/test/libc/calls/reservefd_test.c b/test/libc/calls/reservefd_test.c index 0d958625f..a513cb243 100644 --- a/test/libc/calls/reservefd_test.c +++ b/test/libc/calls/reservefd_test.c @@ -23,7 +23,9 @@ #include "libc/calls/struct/rlimit.h" #include "libc/calls/struct/sigaction.h" #include "libc/errno.h" +#include "libc/intrin/wait0.internal.h" #include "libc/macros.internal.h" +#include "libc/nexgen32e/threaded.h" #include "libc/runtime/internal.h" #include "libc/runtime/stack.h" #include "libc/stdio/rand.h" @@ -39,8 +41,6 @@ #include "libc/testlib/hyperion.h" #include "libc/testlib/testlib.h" #include "libc/thread/spawn.h" -#include "libc/thread/tls.h" -#include "libc/thread/wait0.internal.h" #include "libc/time/struct/tm.h" #include "libc/time/time.h" @@ -61,6 +61,7 @@ void PullSomeZipFilesIntoLinkage(void) { TEST(reservefd, testGrowthOfFdsDataStructure) { int i, n; struct rlimit rlim; + ASSERT_EQ(g_fds.n, OPEN_MAX); n = 1700; // pe '2**16/40' → 1638 (likely value of g_fds.n) if (!getrlimit(RLIMIT_NOFILE, &rlim)) n = MIN(n, rlim.rlim_cur - 3); for (i = 0; i < n; ++i) { diff --git a/test/libc/calls/test.mk b/test/libc/calls/test.mk index a4fbbe871..16c467504 100644 --- a/test/libc/calls/test.mk +++ b/test/libc/calls/test.mk @@ -103,9 +103,6 @@ o/$(MODE)/test/libc/calls/poll_test.com.runs: \ o/$(MODE)/test/libc/calls/fcntl_test.com.runs: \ private .PLEDGE = stdio rpath wpath cpath fattr proc flock -o/$(MODE)/test/libc/calls/lock_test.com.runs: \ - private .PLEDGE = stdio rpath wpath cpath fattr proc flock - o/$(MODE)/test/libc/calls/openbsd_test.com.runs: \ private .PLEDGE = stdio rpath wpath cpath fattr proc unveil diff --git a/test/libc/stdio/ecvt_test.c b/test/libc/fmt/fcvt_test.c similarity index 100% rename from test/libc/stdio/ecvt_test.c rename to test/libc/fmt/fcvt_test.c diff --git a/test/libc/thread/pthread_barrier_wait_test.c b/test/libc/intrin/pthread_barrier_wait_test.c similarity index 95% rename from test/libc/thread/pthread_barrier_wait_test.c rename to test/libc/intrin/pthread_barrier_wait_test.c index dd9942fed..baa0d4acb 100644 --- a/test/libc/thread/pthread_barrier_wait_test.c +++ b/test/libc/intrin/pthread_barrier_wait_test.c @@ -17,17 +17,16 @@ │ PERFORMANCE OF THIS SOFTWARE. │ ╚─────────────────────────────────────────────────────────────────────────────*/ #include "libc/assert.h" -#include "libc/atomic.h" #include "libc/errno.h" #include "libc/intrin/atomic.h" +#include "libc/intrin/pthread.h" #include "libc/mem/mem.h" #include "libc/runtime/gc.internal.h" #include "libc/testlib/testlib.h" #include "libc/thread/spawn.h" -#include "libc/thread/thread.h" int i, n; -atomic_int p, w; +_Atomic(int) p, w; pthread_barrier_t barrier; int Worker(void *arg, int tid) { @@ -37,6 +36,9 @@ int Worker(void *arg, int tid) { ASSERT_GE(rc, 0); if (rc == PTHREAD_BARRIER_SERIAL_THREAD) { atomic_fetch_add(&p, 1); + ASSERT_EQ(0, barrier.popped); + ASSERT_EQ(0, barrier.waits); + ASSERT_EQ(n, barrier.count); } return 0; } diff --git a/test/libc/intrin/pthread_cond_broadcast_test.c b/test/libc/intrin/pthread_cond_broadcast_test.c new file mode 100644 index 000000000..9d944690d --- /dev/null +++ b/test/libc/intrin/pthread_cond_broadcast_test.c @@ -0,0 +1,123 @@ +/*-*- mode:c;indent-tabs-mode:nil;c-basic-offset:2;tab-width:8;coding:utf-8 -*-│ +│vi: set net ft=c ts=2 sts=2 sw=2 fenc=utf-8 :vi│ +╞══════════════════════════════════════════════════════════════════════════════╡ +│ Copyright 2022 Justine Alexandra Roberts Tunney │ +│ │ +│ Permission to use, copy, modify, and/or distribute this software for │ +│ any purpose with or without fee is hereby granted, provided that the │ +│ above copyright notice and this permission notice appear in all copies. │ +│ │ +│ THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL │ +│ WARRANTIES WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED │ +│ WARRANTIES OF MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE │ +│ AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL │ +│ DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR │ +│ PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER │ +│ TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR │ +│ PERFORMANCE OF THIS SOFTWARE. │ +╚─────────────────────────────────────────────────────────────────────────────*/ +#include "libc/calls/struct/timespec.h" +#include "libc/errno.h" +#include "libc/intrin/atomic.h" +#include "libc/intrin/kprintf.h" +#include "libc/intrin/pthread.h" +#include "libc/intrin/pthread2.h" +#include "libc/mem/mem.h" +#include "libc/runtime/gc.internal.h" +#include "libc/testlib/testlib.h" +#include "libc/thread/spawn.h" + +// TODO(jart): Re-enable me. +#if 0 +_Atomic(int) bReady; +pthread_cond_t bCond; +pthread_mutex_t bMutex; + +int BroadcastWorker(void *arg, int tid) { + ASSERT_EQ(0, pthread_mutex_lock(&bMutex)); + atomic_fetch_add(&bReady, 1); + ASSERT_EQ(0, pthread_cond_wait(&bCond, &bMutex)); + ASSERT_EQ(0, pthread_mutex_unlock(&bMutex)); + return 0; +} + +TEST(pthread_cond_broadcast, test) { + pthread_condattr_t cAttr; + pthread_mutexattr_t mAttr; + ASSERT_EQ(0, pthread_mutexattr_init(&mAttr)); + ASSERT_EQ(0, pthread_mutexattr_settype(&mAttr, PTHREAD_MUTEX_ERRORCHECK)); + ASSERT_EQ(0, pthread_mutex_init(&bMutex, &mAttr)); + ASSERT_EQ(0, pthread_condattr_init(&cAttr)); + ASSERT_EQ(0, pthread_condattr_setpshared(&cAttr, PTHREAD_PROCESS_PRIVATE)); + ASSERT_EQ(0, pthread_cond_init(&bCond, &cAttr)); + int i, n = 16; + struct spawn *t = gc(malloc(sizeof(struct spawn) * n)); + for (i = 0; i < n; ++i) { + ASSERT_SYS(0, 0, _spawn(BroadcastWorker, 0, t + i)); + } + for (;;) { + if (atomic_load_explicit(&bReady, memory_order_relaxed) == n) { + break; + } else { + pthread_yield(); + } + } + ASSERT_EQ(0, pthread_mutex_lock(&bMutex)); + ASSERT_EQ(0, pthread_cond_broadcast(&bCond)); + ASSERT_EQ(0, pthread_mutex_unlock(&bMutex)); + for (i = 0; i < n; ++i) { + EXPECT_SYS(0, 0, _join(t + i)); + } + ASSERT_EQ(0, pthread_mutex_destroy(&bMutex)); + ASSERT_EQ(0, pthread_cond_destroy(&bCond)); +} + +TEST(pthread_cond_timedwait, testTimeoutInPast_timesOutImmediately) { + struct timespec t = {100000}; + pthread_cond_t c = PTHREAD_COND_INITIALIZER; + pthread_mutex_t m = {PTHREAD_MUTEX_ERRORCHECK}; + ASSERT_EQ(0, pthread_mutex_lock(&m)); + ASSERT_EQ(ETIMEDOUT, pthread_cond_timedwait(&c, &m, &t)); + ASSERT_EQ(0, pthread_mutex_unlock(&m)); +} + +_Atomic(int) tReady; +pthread_cond_t tCond; +pthread_mutex_t tMutex; + +int TimedWorker(void *arg, int tid) { + struct timespec ts; + ASSERT_EQ(0, pthread_mutex_lock(&tMutex)); + atomic_fetch_add(&tReady, 1); + ts = _timespec_add(_timespec_mono(), _timespec_frommillis(30000)); + ASSERT_EQ(0, pthread_cond_timedwait(&tCond, &tMutex, &ts)); + ASSERT_EQ(0, pthread_mutex_unlock(&tMutex)); + return 0; +} + +TEST(pthread_cond_timedwait, testThirtySeconds_doesntTimeOut) { + pthread_condattr_t cAttr; + pthread_mutexattr_t mAttr; + ASSERT_EQ(0, pthread_mutexattr_init(&mAttr)); + ASSERT_EQ(0, pthread_mutexattr_settype(&mAttr, PTHREAD_MUTEX_ERRORCHECK)); + ASSERT_EQ(0, pthread_mutex_init(&tMutex, &mAttr)); + ASSERT_EQ(0, pthread_condattr_init(&cAttr)); + ASSERT_EQ(0, pthread_condattr_setpshared(&cAttr, PTHREAD_PROCESS_PRIVATE)); + ASSERT_EQ(0, pthread_cond_init(&tCond, &cAttr)); + struct spawn t; + ASSERT_SYS(0, 0, _spawn(TimedWorker, 0, &t)); + for (;;) { + if (atomic_load_explicit(&tReady, memory_order_relaxed)) { + break; + } else { + pthread_yield(); + } + } + ASSERT_EQ(0, pthread_mutex_lock(&tMutex)); + ASSERT_EQ(0, pthread_cond_signal(&tCond)); + ASSERT_EQ(0, pthread_mutex_unlock(&tMutex)); + EXPECT_SYS(0, 0, _join(&t)); + ASSERT_EQ(0, pthread_mutex_destroy(&tMutex)); + ASSERT_EQ(0, pthread_cond_destroy(&tCond)); +} +#endif diff --git a/test/libc/intrin/pthread_mutex_lock2_test.c b/test/libc/intrin/pthread_mutex_lock2_test.c index c582c6167..e8d8794f9 100644 --- a/test/libc/intrin/pthread_mutex_lock2_test.c +++ b/test/libc/intrin/pthread_mutex_lock2_test.c @@ -16,25 +16,23 @@ │ TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR │ │ PERFORMANCE OF THIS SOFTWARE. │ ╚─────────────────────────────────────────────────────────────────────────────*/ -#include "libc/atomic.h" #include "libc/calls/calls.h" #include "libc/calls/struct/timespec.h" #include "libc/intrin/atomic.h" +#include "libc/intrin/pthread.h" #include "libc/mem/mem.h" #include "libc/runtime/gc.internal.h" #include "libc/testlib/ezbench.h" #include "libc/testlib/testlib.h" #include "libc/thread/posixthread.internal.h" #include "libc/thread/spawn.h" -#include "libc/thread/thread.h" -#include "third_party/nsync/mu.h" int THREADS = 16; int ITERATIONS = 100; int count; -atomic_int started; -atomic_int finished; +_Atomic(int) started; +_Atomic(int) finished; pthread_mutex_t lock; pthread_mutexattr_t attr; @@ -110,50 +108,29 @@ void BenchLockUnlock(pthread_mutex_t *m) { pthread_mutex_unlock(m); } -void BenchLockUnlockNsync(nsync_mu *m) { - nsync_mu_lock(m); - nsync_mu_unlock(m); -} - BENCH(pthread_mutex_lock, bench_uncontended) { { pthread_spinlock_t s = {0}; EZBENCH2("spin 1x", donothing, BenchSpinUnspin(&s)); } { - nsync_mu m = {0}; - EZBENCH2("nsync 1x", donothing, BenchLockUnlockNsync(&m)); - } - { - pthread_mutex_t m; - pthread_mutexattr_t attr; - pthread_mutexattr_init(&attr); - pthread_mutexattr_settype(&attr, PTHREAD_MUTEX_NORMAL); - pthread_mutex_init(&m, &attr); + pthread_mutex_t m = {PTHREAD_MUTEX_NORMAL}; EZBENCH2("normal 1x", donothing, BenchLockUnlock(&m)); } { - pthread_mutex_t m; - pthread_mutexattr_t attr; - pthread_mutexattr_init(&attr); - pthread_mutexattr_settype(&attr, PTHREAD_MUTEX_RECURSIVE); - pthread_mutex_init(&m, &attr); + pthread_mutex_t m = {PTHREAD_MUTEX_RECURSIVE}; EZBENCH2("recursive 1x", donothing, BenchLockUnlock(&m)); } { - pthread_mutex_t m; - pthread_mutexattr_t attr; - pthread_mutexattr_init(&attr); - pthread_mutexattr_settype(&attr, PTHREAD_MUTEX_ERRORCHECK); - pthread_mutex_init(&m, &attr); + pthread_mutex_t m = {PTHREAD_MUTEX_ERRORCHECK}; EZBENCH2("errorcheck 1x", donothing, BenchLockUnlock(&m)); } } struct SpinContentionArgs { pthread_spinlock_t *spin; - atomic_char done; - atomic_char ready; + _Atomic(char) done; + _Atomic(char) ready; }; int SpinContentionWorker(void *arg, int tid) { @@ -168,8 +145,8 @@ int SpinContentionWorker(void *arg, int tid) { struct MutexContentionArgs { pthread_mutex_t *mutex; - atomic_char done; - atomic_char ready; + _Atomic(char) done; + _Atomic(char) ready; }; int MutexContentionWorker(void *arg, int tid) { @@ -182,22 +159,6 @@ int MutexContentionWorker(void *arg, int tid) { return 0; } -struct NsyncContentionArgs { - nsync_mu *nsync; - atomic_char done; - atomic_char ready; -}; - -int NsyncContentionWorker(void *arg, int tid) { - struct NsyncContentionArgs *a = arg; - while (!atomic_load_explicit(&a->done, memory_order_relaxed)) { - nsync_mu_lock(a->nsync); - atomic_store_explicit(&a->ready, 1, memory_order_relaxed); - nsync_mu_unlock(a->nsync); - } - return 0; -} - BENCH(pthread_mutex_lock, bench_contended) { struct spawn t; { @@ -210,20 +171,7 @@ BENCH(pthread_mutex_lock, bench_contended) { _join(&t); } { - nsync_mu m = {0}; - struct NsyncContentionArgs a = {&m}; - _spawn(NsyncContentionWorker, &a, &t); - while (!a.ready) sched_yield(); - EZBENCH2("nsync 2x", donothing, BenchLockUnlockNsync(&m)); - a.done = true; - _join(&t); - } - { - pthread_mutex_t m; - pthread_mutexattr_t attr; - pthread_mutexattr_init(&attr); - pthread_mutexattr_settype(&attr, PTHREAD_MUTEX_NORMAL); - pthread_mutex_init(&m, &attr); + pthread_mutex_t m = {PTHREAD_MUTEX_NORMAL}; struct MutexContentionArgs a = {&m}; _spawn(MutexContentionWorker, &a, &t); while (!a.ready) sched_yield(); @@ -232,11 +180,7 @@ BENCH(pthread_mutex_lock, bench_contended) { _join(&t); } { - pthread_mutex_t m; - pthread_mutexattr_t attr; - pthread_mutexattr_init(&attr); - pthread_mutexattr_settype(&attr, PTHREAD_MUTEX_RECURSIVE); - pthread_mutex_init(&m, &attr); + pthread_mutex_t m = {PTHREAD_MUTEX_RECURSIVE}; struct MutexContentionArgs a = {&m}; _spawn(MutexContentionWorker, &a, &t); while (!a.ready) sched_yield(); @@ -245,11 +189,7 @@ BENCH(pthread_mutex_lock, bench_contended) { _join(&t); } { - pthread_mutex_t m; - pthread_mutexattr_t attr; - pthread_mutexattr_init(&attr); - pthread_mutexattr_settype(&attr, PTHREAD_MUTEX_ERRORCHECK); - pthread_mutex_init(&m, &attr); + pthread_mutex_t m = {PTHREAD_MUTEX_ERRORCHECK}; struct MutexContentionArgs a = {&m}; _spawn(MutexContentionWorker, &a, &t); while (!a.ready) sched_yield(); diff --git a/test/libc/intrin/pthread_mutex_lock_test.c b/test/libc/intrin/pthread_mutex_lock_test.c index 396bb9319..97e4f515a 100644 --- a/test/libc/intrin/pthread_mutex_lock_test.c +++ b/test/libc/intrin/pthread_mutex_lock_test.c @@ -17,15 +17,18 @@ │ PERFORMANCE OF THIS SOFTWARE. │ ╚─────────────────────────────────────────────────────────────────────────────*/ #include "libc/assert.h" -#include "libc/atomic.h" #include "libc/calls/calls.h" #include "libc/calls/state.internal.h" #include "libc/calls/strace.internal.h" #include "libc/errno.h" +#include "libc/intrin/futex.internal.h" +#include "libc/intrin/pthread.h" +#include "libc/intrin/wait0.internal.h" #include "libc/log/check.h" #include "libc/macros.internal.h" #include "libc/math.h" #include "libc/mem/mem.h" +#include "libc/nexgen32e/threaded.h" #include "libc/runtime/gc.internal.h" #include "libc/runtime/internal.h" #include "libc/runtime/runtime.h" @@ -38,15 +41,13 @@ #include "libc/testlib/testlib.h" #include "libc/thread/spawn.h" #include "libc/thread/thread.h" -#include "libc/thread/tls.h" -#include "libc/thread/wait0.internal.h" #define THREADS 8 #define ITERATIONS 512 int count; -atomic_int started; -atomic_int finished; +_Atomic(int) started; +_Atomic(int) finished; pthread_mutex_t mylock; pthread_spinlock_t slock; struct spawn th[THREADS]; diff --git a/test/libc/intrin/pthread_once_test.c b/test/libc/intrin/pthread_once_test.c index e51f7a574..9032724a8 100644 --- a/test/libc/intrin/pthread_once_test.c +++ b/test/libc/intrin/pthread_once_test.c @@ -16,19 +16,17 @@ │ TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR │ │ PERFORMANCE OF THIS SOFTWARE. │ ╚─────────────────────────────────────────────────────────────────────────────*/ -#include "libc/atomic.h" #include "libc/intrin/atomic.h" +#include "libc/intrin/pthread.h" #include "libc/mem/mem.h" #include "libc/runtime/gc.internal.h" #include "libc/testlib/testlib.h" #include "libc/thread/spawn.h" -#include "libc/thread/thread.h" int i, n; struct spawn *t; -atomic_int x, y; +_Atomic(int) x, y; pthread_barrier_t b; -static pthread_once_t once = PTHREAD_ONCE_INIT; void InitFactory(void) { ASSERT_EQ(0, atomic_load(&x)); @@ -36,6 +34,7 @@ void InitFactory(void) { } int Worker(void *arg, int tid) { + static pthread_once_t once = PTHREAD_ONCE_INIT; pthread_barrier_wait(&b); ASSERT_EQ(0, pthread_once(&once, InitFactory)); ASSERT_EQ(1, atomic_load(&y)); @@ -46,11 +45,10 @@ int Worker(void *arg, int tid) { TEST(pthread_once, test) { n = 32; x = y = 0; - ASSERT_EQ(0, pthread_barrier_init(&b, 0, n)); + pthread_barrier_init(&b, 0, n); t = gc(malloc(sizeof(struct spawn) * n)); for (i = 0; i < n; ++i) ASSERT_SYS(0, 0, _spawn(Worker, 0, t + i)); for (i = 0; i < n; ++i) EXPECT_SYS(0, 0, _join(t + i)); ASSERT_EQ(n, atomic_load(&x)); ASSERT_EQ(1, atomic_load(&y)); - ASSERT_EQ(0, pthread_barrier_destroy(&b)); } diff --git a/test/libc/thread/pthread_rwlock_rdlock_test.c b/test/libc/intrin/pthread_rwlock_rdlock_test.c similarity index 83% rename from test/libc/thread/pthread_rwlock_rdlock_test.c rename to test/libc/intrin/pthread_rwlock_rdlock_test.c index 28f4eab48..6eaf59a9a 100644 --- a/test/libc/thread/pthread_rwlock_rdlock_test.c +++ b/test/libc/intrin/pthread_rwlock_rdlock_test.c @@ -16,28 +16,27 @@ │ TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR │ │ PERFORMANCE OF THIS SOFTWARE. │ ╚─────────────────────────────────────────────────────────────────────────────*/ -#include "libc/atomic.h" +#include "libc/intrin/pthread.h" #include "libc/mem/mem.h" -#include "libc/runtime/gc.h" +#include "libc/runtime/gc.internal.h" #include "libc/testlib/testlib.h" #include "libc/thread/spawn.h" -#include "libc/thread/thread.h" #define ITERATIONS 50000 #define READERS 8 #define WRITERS 2 -atomic_int reads; -atomic_int writes; +_Atomic(int) reads; +_Atomic(int) writes; pthread_rwlock_t lock; pthread_barrier_t barrier; int Reader(void *arg, int tid) { pthread_barrier_wait(&barrier); for (int i = 0; i < ITERATIONS; ++i) { - ASSERT_EQ(0, pthread_rwlock_rdlock(&lock)); + pthread_rwlock_rdlock(&lock); ++reads; - ASSERT_EQ(0, pthread_rwlock_unlock(&lock)); + pthread_rwlock_unlock(&lock); } return 0; } @@ -45,17 +44,17 @@ int Reader(void *arg, int tid) { int Writer(void *arg, int tid) { pthread_barrier_wait(&barrier); for (int i = 0; i < ITERATIONS; ++i) { - ASSERT_EQ(0, pthread_rwlock_wrlock(&lock)); + pthread_rwlock_wrlock(&lock); ++writes; - ASSERT_EQ(0, pthread_rwlock_unlock(&lock)); + pthread_rwlock_unlock(&lock); } return 0; } TEST(pthread_rwlock_rdlock, test) { int i; - struct spawn *t = _gc(malloc(sizeof(struct spawn) * (READERS + WRITERS))); - ASSERT_EQ(0, pthread_barrier_init(&barrier, 0, READERS + WRITERS)); + struct spawn *t = gc(malloc(sizeof(struct spawn) * (READERS + WRITERS))); + pthread_barrier_init(&barrier, 0, READERS + WRITERS); for (i = 0; i < READERS + WRITERS; ++i) { ASSERT_SYS(0, 0, _spawn(i < READERS ? Reader : Writer, 0, t + i)); } @@ -64,5 +63,4 @@ TEST(pthread_rwlock_rdlock, test) { } EXPECT_EQ(READERS * ITERATIONS, reads); EXPECT_EQ(WRITERS * ITERATIONS, writes); - ASSERT_EQ(0, pthread_barrier_destroy(&barrier)); } diff --git a/test/libc/intrin/rand64_test.c b/test/libc/intrin/rand64_test.c index 0c6efa4fd..4251d7a43 100644 --- a/test/libc/intrin/rand64_test.c +++ b/test/libc/intrin/rand64_test.c @@ -25,6 +25,7 @@ #include "libc/limits.h" #include "libc/macros.internal.h" #include "libc/mem/mem.h" +#include "libc/nexgen32e/threaded.h" #include "libc/runtime/internal.h" #include "libc/runtime/stack.h" #include "libc/stdio/rand.h" @@ -43,7 +44,7 @@ #define ENTRIES 1024 volatile uint64_t A[THREADS * ENTRIES]; -pthread_barrier_t barrier; +pthread_barrier_t barrier = PTHREAD_BARRIER_INITIALIZER; void SetUpOnce(void) { __enable_threads(); @@ -93,7 +94,7 @@ TEST(rand64, testThreadSafety_doesntProduceIdenticalValues) { sigemptyset(&ss); sigaddset(&ss, SIGCHLD); EXPECT_EQ(0, sigprocmask(SIG_BLOCK, &ss, &oldss)); - ASSERT_EQ(0, pthread_barrier_init(&barrier, 0, THREADS)); + pthread_barrier_init(&barrier, 0, THREADS); for (i = 0; i < THREADS; ++i) { ASSERT_SYS(0, 0, _spawn(Thrasher, (void *)(intptr_t)i, th + i)); } @@ -109,5 +110,4 @@ TEST(rand64, testThreadSafety_doesntProduceIdenticalValues) { EXPECT_NE(A[i], A[j], "i=%d j=%d", i, j); } } - ASSERT_EQ(0, pthread_barrier_destroy(&barrier)); } diff --git a/test/libc/intrin/test.mk b/test/libc/intrin/test.mk index 48ce2837a..5eb021c64 100644 --- a/test/libc/intrin/test.mk +++ b/test/libc/intrin/test.mk @@ -39,8 +39,7 @@ TEST_LIBC_INTRIN_DIRECTDEPS = \ LIBC_TINYMATH \ LIBC_X \ TOOL_VIZ_LIB \ - THIRD_PARTY_COMPILER_RT \ - THIRD_PARTY_NSYNC + THIRD_PARTY_COMPILER_RT TEST_LIBC_INTRIN_DEPS := \ $(call uniq,$(foreach x,$(TEST_LIBC_INTRIN_DIRECTDEPS),$($(x)))) diff --git a/test/libc/nexgen32e/stackrw_test.c b/test/libc/nexgen32e/stackrw_test.c index 4f55e45f6..97f85d28c 100644 --- a/test/libc/nexgen32e/stackrw_test.c +++ b/test/libc/nexgen32e/stackrw_test.c @@ -17,8 +17,6 @@ │ PERFORMANCE OF THIS SOFTWARE. │ ╚─────────────────────────────────────────────────────────────────────────────*/ #include "libc/calls/struct/sigaction.h" -#include "libc/calls/ucontext.h" -#include "libc/dce.h" #include "libc/runtime/runtime.h" #include "libc/sysv/consts/sa.h" #include "libc/sysv/consts/sig.h" @@ -30,16 +28,15 @@ jmp_buf jb; -void EscapeSegfault(int sig, struct siginfo *si, void *vctx) { +void EscapeSegfault(int sig) { longjmp(jb, 666); } TEST(xstack, test) { - if (IsXnu()) return; // TODO(jart): what's up with xnu in MODE=tiny? struct sigaction old[2]; struct sigaction sa = { - .sa_sigaction = EscapeSegfault, - .sa_flags = SA_SIGINFO, + .sa_handler = EscapeSegfault, + .sa_flags = SA_NODEFER, }; sigaction(SIGSEGV, &sa, old + 0); sigaction(SIGBUS, &sa, old + 1); diff --git a/test/libc/release/test.mk b/test/libc/release/test.mk index 23c3393e1..502f01fa2 100644 --- a/test/libc/release/test.mk +++ b/test/libc/release/test.mk @@ -1,7 +1,6 @@ #-*-mode:makefile-gmake;indent-tabs-mode:t;tab-width:8;coding:utf-8-*-┐ #───vi: set et ft=make ts=8 tw=8 fenc=utf-8 :vi───────────────────────┘ -o/$(MODE)/test/libc/release/cosmopolitan.zip: .UNSANDBOXED = 1 o/$(MODE)/test/libc/release/cosmopolitan.zip: \ o/cosmopolitan.h \ o/$(MODE)/ape/public/ape.lds \ diff --git a/test/libc/runtime/arch_prctl_test.c b/test/libc/runtime/arch_prctl_test.c index a66123e20..6c1e9cda9 100644 --- a/test/libc/runtime/arch_prctl_test.c +++ b/test/libc/runtime/arch_prctl_test.c @@ -24,7 +24,7 @@ #include "libc/intrin/fsgsbase.h" #include "libc/intrin/kprintf.h" #include "libc/intrin/segmentation.h" -#include "libc/thread/tls.h" +#include "libc/nexgen32e/threaded.h" #include "libc/nt/version.h" #include "libc/sysv/consts/sa.h" #include "libc/sysv/consts/sig.h" diff --git a/test/libc/thread/clone_test.c b/test/libc/runtime/clone_test.c similarity index 86% rename from test/libc/thread/clone_test.c rename to test/libc/runtime/clone_test.c index 4457364fd..a6d9c2cce 100644 --- a/test/libc/thread/clone_test.c +++ b/test/libc/runtime/clone_test.c @@ -16,14 +16,16 @@ │ TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR │ │ PERFORMANCE OF THIS SOFTWARE. │ ╚─────────────────────────────────────────────────────────────────────────────*/ -#include "libc/atomic.h" #include "libc/calls/calls.h" #include "libc/calls/struct/timespec.h" #include "libc/dce.h" #include "libc/errno.h" +#include "libc/intrin/futex.internal.h" +#include "libc/intrin/wait0.internal.h" #include "libc/log/backtrace.internal.h" #include "libc/macros.internal.h" #include "libc/mem/mem.h" +#include "libc/nexgen32e/gettls.h" #include "libc/nexgen32e/nexgen32e.h" #include "libc/runtime/internal.h" #include "libc/runtime/runtime.h" @@ -40,19 +42,26 @@ #include "libc/testlib/ezbench.h" #include "libc/testlib/testlib.h" #include "libc/thread/spawn.h" -#include "libc/thread/tls.h" -#include "libc/thread/tls2.h" -#include "libc/thread/wait0.internal.h" #include "libc/time/time.h" int x, me, tid; -atomic_int thechilde; +_Atomic(int) thechilde; void SetUpOnce(void) { __enable_threads(); ASSERT_SYS(0, 0, pledge("stdio rpath", 0)); } +void *__initialize_tls(char tib[64]) { + if (tib) { + *(intptr_t *)(tib + 0x00) = (intptr_t)tib; + *(intptr_t *)(tib + 0x30) = (intptr_t)tib; + *(int *)(tib + 0x38) = -1; // tid + *(int *)(tib + 0x3c) = 0; + } + return tib; +} + int Hog(void *arg, int tid) { return 0; } @@ -109,7 +118,7 @@ TEST(clone, test1) { //////////////////////////////////////////////////////////////////////////////// // TEST THREADS CAN ISSUE SYSTEM CALLS WITH INDEPENDENT ERRNOS -atomic_int sysbarrier; +_Atomic(int) sysbarrier; int CloneTestSys(void *arg, int tid) { int i, id = (intptr_t)arg; @@ -155,3 +164,14 @@ TEST(clone, tlsSystemCallsErrno_wontClobberMainThreadBecauseTls) { } ASSERT_EQ(0, errno); } + +//////////////////////////////////////////////////////////////////////////////// +// BENCHMARK + +BENCH(clone, bench) { + char *volatile tp; + errno_t *volatile ep; + EZBENCH2("__errno_location", donothing, (ep = __errno_location())); + EZBENCH2("__get_tls_priv", donothing, (tp = __get_tls_privileged())); + EZBENCH2("__get_tls", donothing, (tp = __get_tls())); +} diff --git a/test/libc/runtime/tls_test.c b/test/libc/runtime/tls_test.c index 69bf48225..59e2ad501 100644 --- a/test/libc/runtime/tls_test.c +++ b/test/libc/runtime/tls_test.c @@ -16,39 +16,12 @@ │ TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR │ │ PERFORMANCE OF THIS SOFTWARE. │ ╚─────────────────────────────────────────────────────────────────────────────*/ -#include "libc/dce.h" -#include "libc/intrin/asan.internal.h" #include "libc/testlib/testlib.h" -#include "libc/thread/thread.h" -#include "libc/thread/tls.h" -#define A TLS_ALIGNMENT - -long z = 2; -pthread_t t; -_Thread_local long x; -_Thread_local long y[1] = {40}; -_Alignas(A) _Thread_local long a; - -noubsan void *Worker(void *arg) { - ASSERT_EQ(42, x + y[0] + z); - ASSERT_EQ(0, (intptr_t)&a & (A - 1)); - if (IsAsan()) { - ASSERT_EQ(kAsanProtected, __asan_check(y + 1, sizeof(long)).kind); - } - return 0; -} +_Thread_local int x; +_Thread_local int y = 40; +int z = 2; TEST(tls, test) { - ASSERT_EQ(A, _Alignof(a)); - ASSERT_EQ(0, sizeof(struct CosmoTib) % A); - ASSERT_EQ(0, (intptr_t)__get_tls() & (A - 1)); - EXPECT_EQ(42, x + y[0] + z); - y[0] = 666; - ASSERT_EQ(0, (intptr_t)&a & (A - 1)); - ASSERT_EQ(0, pthread_create(&t, 0, Worker, 0)); - ASSERT_EQ(0, pthread_join(t, 0)); - if (IsAsan()) { - ASSERT_EQ(kAsanProtected, __asan_check(y + 1, sizeof(long)).kind); - } + EXPECT_EQ(42, x + y + z); } diff --git a/test/libc/stdio/dtoa_test.c b/test/libc/stdio/dtoa_test.c index 0ecebcccc..7b2cc638f 100644 --- a/test/libc/stdio/dtoa_test.c +++ b/test/libc/stdio/dtoa_test.c @@ -20,6 +20,7 @@ #include "libc/calls/struct/sched_param.h" #include "libc/dce.h" #include "libc/fmt/fmt.h" +#include "libc/intrin/wait0.internal.h" #include "libc/macros.internal.h" #include "libc/math.h" #include "libc/mem/mem.h" @@ -34,7 +35,6 @@ #include "libc/sysv/consts/sched.h" #include "libc/testlib/testlib.h" #include "libc/thread/spawn.h" -#include "libc/thread/wait0.internal.h" #include "libc/x/x.h" #define DUB(i) (union Dub){i}.x diff --git a/test/libc/stdio/fputc_test.c b/test/libc/stdio/fputc_test.c index e11599b06..8b02a0de3 100644 --- a/test/libc/stdio/fputc_test.c +++ b/test/libc/stdio/fputc_test.c @@ -18,7 +18,7 @@ ╚─────────────────────────────────────────────────────────────────────────────*/ #include "libc/calls/calls.h" #include "libc/runtime/internal.h" -#include "libc/stdio/lock.internal.h" +#include "libc/stdio/lock.h" #include "libc/stdio/stdio.h" #include "libc/str/str.h" #include "libc/testlib/ezbench.h" diff --git a/test/libc/test.mk b/test/libc/test.mk index 4abd1a1e0..ef908621a 100644 --- a/test/libc/test.mk +++ b/test/libc/test.mk @@ -19,5 +19,4 @@ o/$(MODE)/test/libc: \ o/$(MODE)/test/libc/time \ o/$(MODE)/test/libc/tinymath \ o/$(MODE)/test/libc/x \ - o/$(MODE)/test/libc/zipos \ o/$(MODE)/test/libc/xed diff --git a/test/libc/thread/nsync_test.c b/test/libc/thread/nsync_test.c deleted file mode 100644 index 6c27d1ef7..000000000 --- a/test/libc/thread/nsync_test.c +++ /dev/null @@ -1,119 +0,0 @@ -/*-*- mode:c;indent-tabs-mode:nil;c-basic-offset:2;tab-width:8;coding:utf-8 -*-│ -│vi: set net ft=c ts=2 sts=2 sw=2 fenc=utf-8 :vi│ -╞══════════════════════════════════════════════════════════════════════════════╡ -│ Copyright 2022 Justine Alexandra Roberts Tunney │ -│ │ -│ Permission to use, copy, modify, and/or distribute this software for │ -│ any purpose with or without fee is hereby granted, provided that the │ -│ above copyright notice and this permission notice appear in all copies. │ -│ │ -│ THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL │ -│ WARRANTIES WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED │ -│ WARRANTIES OF MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE │ -│ AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL │ -│ DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR │ -│ PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER │ -│ TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR │ -│ PERFORMANCE OF THIS SOFTWARE. │ -╚─────────────────────────────────────────────────────────────────────────────*/ -#include "libc/errno.h" -#include "libc/testlib/testlib.h" -#include "libc/thread/thread.h" -#include "third_party/nsync/cv.h" -#include "third_party/nsync/mu.h" - -int pos; -int count; -int limit; -long data[1000]; -nsync_mu mu; -nsync_cv non_full; -nsync_cv non_empty; - -int Put(long v, nsync_time abs_deadline) { - int err, added = 0, wake = 0; - nsync_mu_lock(&mu); - while (count == limit) { - if ((err = nsync_cv_wait_with_deadline(&non_full, &mu, abs_deadline, 0))) { - ASSERT_EQ(ETIMEDOUT, err); - ASSERT_NE(0, nsync_time_cmp(nsync_time_no_deadline, abs_deadline)); - } - } - if (count != limit) { - int i = pos + count; - if (limit <= i) i -= limit; - data[i] = v; - if (count == 0) wake = 1; - count++; - added = 1; - } - nsync_mu_unlock(&mu); - if (wake) nsync_cv_broadcast(&non_empty); - return added; -} - -long Get(nsync_time abs_deadline) { - long err, v = 0; - nsync_mu_lock(&mu); - while (!count) { - if ((err = nsync_cv_wait_with_deadline(&non_empty, &mu, abs_deadline, 0))) { - ASSERT_EQ(ETIMEDOUT, err); - ASSERT_NE(0, nsync_time_cmp(nsync_time_no_deadline, abs_deadline)); - } - } - if (count) { - v = data[pos]; - data[pos] = 0; - if (count == limit) { - nsync_cv_broadcast(&non_full); - } - pos++; - count--; - if (pos == limit) { - pos = 0; - } - } - nsync_mu_unlock(&mu); - return v; -} - -#define N 10000 - -void *Producer(void *arg) { - for (int i = 0; i < N; i++) { - ASSERT_EQ(1, Put((i + 1) * 3, nsync_time_no_deadline)); - } - return 0; -} - -void *Consumer(void *arg) { - for (int i = 0; i < N; i++) { - ASSERT_EQ((i + 1) * 3, Get(nsync_time_no_deadline)); - } - return 0; -} - -TEST(cond, test) { - pthread_t t; - nsync_mu_init(&mu); - - limit = 1; - ASSERT_EQ(0, pthread_create(&t, 0, Producer, 0)); - Consumer(0); - ASSERT_EQ(0, pthread_join(t, 0)); - - limit = 10; - ASSERT_EQ(0, pthread_create(&t, 0, Producer, 0)); - Consumer(0); - ASSERT_EQ(0, pthread_join(t, 0)); - - limit = 100; - ASSERT_EQ(0, pthread_create(&t, 0, Producer, 0)); - Consumer(0); - ASSERT_EQ(0, pthread_join(t, 0)); - - limit = 1000; - ASSERT_EQ(0, pthread_create(&t, 0, Producer, 0)); - Consumer(0); - ASSERT_EQ(0, pthread_join(t, 0)); -} diff --git a/test/libc/thread/pthread_cond_signal_test.c b/test/libc/thread/pthread_cond_signal_test.c deleted file mode 100644 index f72e77abd..000000000 --- a/test/libc/thread/pthread_cond_signal_test.c +++ /dev/null @@ -1,118 +0,0 @@ -/*-*- mode:c;indent-tabs-mode:nil;c-basic-offset:2;tab-width:8;coding:utf-8 -*-│ -│vi: set net ft=c ts=2 sts=2 sw=2 fenc=utf-8 :vi│ -╞══════════════════════════════════════════════════════════════════════════════╡ -│ Copyright 2022 Justine Alexandra Roberts Tunney │ -│ │ -│ Permission to use, copy, modify, and/or distribute this software for │ -│ any purpose with or without fee is hereby granted, provided that the │ -│ above copyright notice and this permission notice appear in all copies. │ -│ │ -│ THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL │ -│ WARRANTIES WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED │ -│ WARRANTIES OF MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE │ -│ AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL │ -│ DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR │ -│ PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER │ -│ TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR │ -│ PERFORMANCE OF THIS SOFTWARE. │ -╚─────────────────────────────────────────────────────────────────────────────*/ -#include "libc/calls/struct/timespec.h" -#include "libc/errno.h" -#include "libc/testlib/testlib.h" -#include "libc/thread/thread.h" -#include "libc/thread/thread2.h" - -int pos; -int count; -int limit; -long data[1000]; -pthread_mutex_t mu; -pthread_cond_t non_full; -pthread_cond_t non_empty; - -int Put(long v, struct timespec *abs_deadline) { - int err, added = 0, wake = 0; - pthread_mutex_lock(&mu); - while (count == limit) { - if ((err = pthread_cond_timedwait(&non_full, &mu, abs_deadline))) { - ASSERT_EQ(ETIMEDOUT, err); - ASSERT_NE(NULL, abs_deadline); - } - } - if (count != limit) { - int i = pos + count; - if (limit <= i) i -= limit; - data[i] = v; - if (count == 0) wake = 1; - count++; - added = 1; - } - pthread_mutex_unlock(&mu); - if (wake) pthread_cond_broadcast(&non_empty); - return added; -} - -long Get(struct timespec *abs_deadline) { - long err, v = 0; - pthread_mutex_lock(&mu); - while (!count) { - if ((err = pthread_cond_timedwait(&non_empty, &mu, abs_deadline))) { - ASSERT_EQ(ETIMEDOUT, err); - ASSERT_NE(NULL, abs_deadline); - } - } - if (count) { - v = data[pos]; - data[pos] = 0; - if (count == limit) { - pthread_cond_broadcast(&non_full); - } - pos++; - count--; - if (pos == limit) { - pos = 0; - } - } - pthread_mutex_unlock(&mu); - return v; -} - -#define N 10000 - -void *Producer(void *arg) { - for (int i = 0; i < N; i++) { - ASSERT_EQ(1, Put((i + 1) * 3, 0)); - } - return 0; -} - -void *Consumer(void *arg) { - for (int i = 0; i < N; i++) { - ASSERT_EQ((i + 1) * 3, Get(0)); - } - return 0; -} - -TEST(cond, test) { - pthread_t t; - - limit = 1; - ASSERT_EQ(0, pthread_create(&t, 0, Producer, 0)); - Consumer(0); - ASSERT_EQ(0, pthread_join(t, 0)); - - limit = 10; - ASSERT_EQ(0, pthread_create(&t, 0, Producer, 0)); - Consumer(0); - ASSERT_EQ(0, pthread_join(t, 0)); - - limit = 100; - ASSERT_EQ(0, pthread_create(&t, 0, Producer, 0)); - Consumer(0); - ASSERT_EQ(0, pthread_join(t, 0)); - - limit = 1000; - ASSERT_EQ(0, pthread_create(&t, 0, Producer, 0)); - Consumer(0); - ASSERT_EQ(0, pthread_join(t, 0)); -} diff --git a/test/libc/thread/pthread_create_test.c b/test/libc/thread/pthread_create_test.c index 96566f673..421761208 100644 --- a/test/libc/thread/pthread_create_test.c +++ b/test/libc/thread/pthread_create_test.c @@ -22,9 +22,10 @@ #include "libc/dce.h" #include "libc/errno.h" #include "libc/intrin/kprintf.h" +#include "libc/intrin/pthread.h" +#include "libc/intrin/pthread2.h" #include "libc/macros.internal.h" #include "libc/mem/mem.h" -#include "libc/nexgen32e/nexgen32e.h" #include "libc/runtime/runtime.h" #include "libc/runtime/stack.h" #include "libc/sysv/consts/prot.h" @@ -35,21 +36,19 @@ #include "libc/testlib/subprocess.h" #include "libc/testlib/testlib.h" #include "libc/thread/thread.h" -#include "libc/thread/thread2.h" -void OnUsr1(int sig, struct siginfo *si, void *vctx) { +void OnTrap(int sig, struct siginfo *si, void *vctx) { struct ucontext *ctx = vctx; } void SetUp(void) { - struct sigaction sig = {.sa_sigaction = OnUsr1, .sa_flags = SA_SIGINFO}; - sigaction(SIGUSR1, &sig, 0); + struct sigaction sig = {.sa_sigaction = OnTrap, .sa_flags = SA_SIGINFO}; + sigaction(SIGTRAP, &sig, 0); } void TriggerSignal(void) { sched_yield(); - /* kprintf("raising at %p\n", __builtin_frame_address(0)); */ - raise(SIGUSR1); + DebugBreak(); sched_yield(); } @@ -68,7 +67,6 @@ TEST(pthread_create, testCreateReturnJoin) { } static void *IncExit(void *arg) { - CheckStackIsAligned(); TriggerSignal(); pthread_exit((void *)((uintptr_t)arg + 1)); } diff --git a/test/libc/thread/pthread_key_create_test.c b/test/libc/thread/pthread_key_create_test.c index ced6cf725..15cdc4bd2 100644 --- a/test/libc/thread/pthread_key_create_test.c +++ b/test/libc/thread/pthread_key_create_test.c @@ -16,7 +16,9 @@ │ TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR │ │ PERFORMANCE OF THIS SOFTWARE. │ ╚─────────────────────────────────────────────────────────────────────────────*/ +#include "libc/dce.h" #include "libc/mem/mem.h" +#include "libc/runtime/internal.h" #include "libc/testlib/testlib.h" #include "libc/thread/thread.h" diff --git a/test/libc/thread/pthread_setname_np_test.c b/test/libc/thread/pthread_setname_np_test.c index 16154b44d..e39226e75 100644 --- a/test/libc/thread/pthread_setname_np_test.c +++ b/test/libc/thread/pthread_setname_np_test.c @@ -16,9 +16,9 @@ │ TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR │ │ PERFORMANCE OF THIS SOFTWARE. │ ╚─────────────────────────────────────────────────────────────────────────────*/ -#include "libc/atomic.h" #include "libc/dce.h" #include "libc/intrin/atomic.h" +#include "libc/intrin/pthread.h" #include "libc/testlib/testlib.h" #include "libc/thread/thread.h" @@ -63,7 +63,7 @@ TEST(pthread_setname_np, GetDefaultName_IsEmptyString) { ASSERT_EQ(0, pthread_join(id, 0)); } -atomic_char sync1, sync2; +_Atomic(char) sync1, sync2; static void *GetNameOfOtherThreadWorker(void *arg) { pthread_setname_np(pthread_self(), "justine"); diff --git a/test/libc/thread/spawn_test.c b/test/libc/thread/spawn_test.c index 084f00341..88193d605 100644 --- a/test/libc/thread/spawn_test.c +++ b/test/libc/thread/spawn_test.c @@ -17,7 +17,6 @@ │ PERFORMANCE OF THIS SOFTWARE. │ ╚─────────────────────────────────────────────────────────────────────────────*/ #include "libc/assert.h" -#include "libc/atomic.h" #include "libc/calls/calls.h" #include "libc/calls/syscall-sysv.internal.h" #include "libc/intrin/atomic.h" @@ -27,8 +26,9 @@ #include "libc/runtime/internal.h" #include "libc/testlib/testlib.h" #include "libc/thread/spawn.h" +#include "libc/thread/thread.h" -atomic_int itworked; +_Atomic(int) itworked; _Thread_local int var; void SetUpOnce(void) { diff --git a/test/libc/thread/test.mk b/test/libc/thread/test.mk index 962a66a96..bbd33da99 100644 --- a/test/libc/thread/test.mk +++ b/test/libc/thread/test.mk @@ -34,8 +34,7 @@ TEST_LIBC_THREAD_DIRECTDEPS = \ LIBC_SYSV \ LIBC_THREAD \ LIBC_TIME \ - LIBC_TESTLIB \ - THIRD_PARTY_NSYNC + LIBC_TESTLIB TEST_LIBC_THREAD_DEPS := \ $(call uniq,$(foreach x,$(TEST_LIBC_THREAD_DIRECTDEPS),$($(x)))) diff --git a/test/libc/x/makedirs_test.c b/test/libc/x/makedirs_test.c index 388006035..d5da76a09 100644 --- a/test/libc/x/makedirs_test.c +++ b/test/libc/x/makedirs_test.c @@ -16,11 +16,11 @@ │ TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR │ │ PERFORMANCE OF THIS SOFTWARE. │ ╚─────────────────────────────────────────────────────────────────────────────*/ +#include "libc/intrin/pthread.h" #include "libc/mem/mem.h" #include "libc/runtime/gc.internal.h" #include "libc/testlib/testlib.h" #include "libc/thread/spawn.h" -#include "libc/thread/thread.h" #include "libc/x/x.h" #define DIR \ @@ -42,5 +42,4 @@ TEST(makedirs, test) { ASSERT_EQ(0, pthread_barrier_init(&barrier, 0, n)); for (i = 0; i < n; ++i) ASSERT_SYS(0, 0, _spawn(Worker, 0, t + i)); for (i = 0; i < n; ++i) EXPECT_SYS(0, 0, _join(t + i)); - ASSERT_EQ(0, pthread_barrier_destroy(&barrier)); } diff --git a/test/libc/zipos/open_test.c b/test/libc/zipos/open_test.c deleted file mode 100644 index 28d97ab6e..000000000 --- a/test/libc/zipos/open_test.c +++ /dev/null @@ -1,56 +0,0 @@ -/*-*- mode:c;indent-tabs-mode:nil;c-basic-offset:2;tab-width:8;coding:utf-8 -*-│ -│vi: set net ft=c ts=2 sts=2 sw=2 fenc=utf-8 :vi│ -╞══════════════════════════════════════════════════════════════════════════════╡ -│ Copyright 2022 Justine Alexandra Roberts Tunney │ -│ │ -│ Permission to use, copy, modify, and/or distribute this software for │ -│ any purpose with or without fee is hereby granted, provided that the │ -│ above copyright notice and this permission notice appear in all copies. │ -│ │ -│ THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL │ -│ WARRANTIES WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED │ -│ WARRANTIES OF MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE │ -│ AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL │ -│ DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR │ -│ PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER │ -│ TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR │ -│ PERFORMANCE OF THIS SOFTWARE. │ -╚─────────────────────────────────────────────────────────────────────────────*/ -#include "libc/calls/calls.h" -#include "libc/mem/mem.h" -#include "libc/runtime/gc.h" -#include "libc/runtime/runtime.h" -#include "libc/str/str.h" -#include "libc/sysv/consts/o.h" -#include "libc/testlib/hyperion.h" -#include "libc/testlib/testlib.h" -#include "libc/thread/spawn.h" -#include "libc/zipos/zipos.h" - -STATIC_YOINK("zip_uri_support"); -STATIC_YOINK("libc/testlib/hyperion.txt"); -STATIC_YOINK("inflate"); -STATIC_YOINK("inflateInit2"); -STATIC_YOINK("inflateEnd"); - -int Worker(void *arg, int tid) { - int i, fd; - char *data; - for (i = 0; i < 20; ++i) { - ASSERT_NE(-1, (fd = open("/zip/libc/testlib/hyperion.txt", O_RDONLY))); - data = malloc(kHyperionSize); - ASSERT_EQ(kHyperionSize, read(fd, data, kHyperionSize)); - ASSERT_EQ(0, memcmp(data, kHyperion, kHyperionSize)); - ASSERT_SYS(0, 0, close(fd)); - free(data); - } - return 0; -} - -TEST(zipos, test) { - int i, n = 16; - struct spawn *t = _gc(malloc(sizeof(struct spawn) * n)); - for (i = 0; i < n; ++i) ASSERT_SYS(0, 0, _spawn(Worker, 0, t + i)); - for (i = 0; i < n; ++i) EXPECT_SYS(0, 0, _join(t + i)); - /* __print_maps(); */ -} diff --git a/test/libc/zipos/test.mk b/test/libc/zipos/test.mk deleted file mode 100644 index 086867e33..000000000 --- a/test/libc/zipos/test.mk +++ /dev/null @@ -1,63 +0,0 @@ -#-*-mode:makefile-gmake;indent-tabs-mode:t;tab-width:8;coding:utf-8-*-┐ -#───vi: set et ft=make ts=8 tw=8 fenc=utf-8 :vi───────────────────────┘ - -PKGS += TEST_LIBC_ZIPOS - -TEST_LIBC_ZIPOS_SRCS := $(wildcard test/libc/zipos/*.c) -TEST_LIBC_ZIPOS_SRCS_TEST = $(filter %_test.c,$(TEST_LIBC_ZIPOS_SRCS)) - -TEST_LIBC_ZIPOS_OBJS = \ - $(TEST_LIBC_ZIPOS_SRCS:%.c=o/$(MODE)/%.o) - -TEST_LIBC_ZIPOS_COMS = \ - $(TEST_LIBC_ZIPOS_SRCS:%.c=o/$(MODE)/%.com) - -TEST_LIBC_ZIPOS_BINS = \ - $(TEST_LIBC_ZIPOS_COMS) \ - $(TEST_LIBC_ZIPOS_COMS:%=%.dbg) - -TEST_LIBC_ZIPOS_TESTS = \ - $(TEST_LIBC_ZIPOS_SRCS_TEST:%.c=o/$(MODE)/%.com.ok) - -TEST_LIBC_ZIPOS_CHECKS = \ - $(TEST_LIBC_ZIPOS_SRCS_TEST:%.c=o/$(MODE)/%.com.runs) - -TEST_LIBC_ZIPOS_DIRECTDEPS = \ - LIBC_CALLS \ - LIBC_FMT \ - LIBC_INTRIN \ - LIBC_MEM \ - LIBC_NEXGEN32E \ - LIBC_RUNTIME \ - LIBC_STR \ - LIBC_STUBS \ - LIBC_THREAD \ - LIBC_SYSV \ - LIBC_ZIPOS \ - LIBC_TIME \ - LIBC_TESTLIB \ - THIRD_PARTY_ZLIB - -TEST_LIBC_ZIPOS_DEPS := \ - $(call uniq,$(foreach x,$(TEST_LIBC_ZIPOS_DIRECTDEPS),$($(x)))) - -o/$(MODE)/test/libc/zipos/zipos.pkg: \ - $(TEST_LIBC_ZIPOS_OBJS) \ - $(foreach x,$(TEST_LIBC_ZIPOS_DIRECTDEPS),$($(x)_A).pkg) - -#o/$(MODE)/libc/testlib/hyperion.txt.zip.o: private ZIPOBJ_FLAGS += -0 - -o/$(MODE)/test/libc/zipos/%.com.dbg: \ - $(TEST_LIBC_ZIPOS_DEPS) \ - o/$(MODE)/test/libc/zipos/%.o \ - o/$(MODE)/test/libc/zipos/zipos.pkg \ - o/$(MODE)/libc/testlib/hyperion.txt.zip.o \ - $(LIBC_TESTMAIN) \ - $(CRT) \ - $(APE_NO_MODIFY_SELF) - @$(APELINK) - -.PHONY: o/$(MODE)/test/libc/zipos -o/$(MODE)/test/libc/zipos: \ - $(TEST_LIBC_ZIPOS_BINS) \ - $(TEST_LIBC_ZIPOS_CHECKS) diff --git a/test/tool/plinko/plinko_test.c b/test/tool/plinko/plinko_test.c index 75fda9923..dfd438767 100644 --- a/test/tool/plinko/plinko_test.c +++ b/test/tool/plinko/plinko_test.c @@ -53,7 +53,6 @@ static const char *const kSauces[] = { char testlib_enable_tmp_setup_teardown_once; void SetUpOnce(void) { - exit(0); // TODO(jart): How can we safely disable TLS with *NSYNC? int fdin, fdout; ASSERT_NE(-1, mkdir("bin", 0755)); ASSERT_NE(-1, (fdin = open("/zip/plinko.com", O_RDONLY))); diff --git a/third_party/awk/awkgram.tab.c b/third_party/awk/awkgram.tab.c index 1acd78ab9..96b3555eb 100644 --- a/third_party/awk/awkgram.tab.c +++ b/third_party/awk/awkgram.tab.c @@ -21,7 +21,7 @@ #line 26 "awkgram.y" #include "libc/calls/calls.h" #include "libc/fmt/fmt.h" -#include "libc/stdio/lock.internal.h" +#include "libc/stdio/lock.h" #include "libc/stdio/stdio.h" #include "libc/stdio/temp.h" #include "libc/mem/alg.h" diff --git a/third_party/awk/awkgram.y b/third_party/awk/awkgram.y index 7c449cabe..04104b35d 100644 --- a/third_party/awk/awkgram.y +++ b/third_party/awk/awkgram.y @@ -25,7 +25,7 @@ THIS SOFTWARE. %{ #include "libc/calls/calls.h" #include "libc/fmt/fmt.h" -#include "libc/stdio/lock.internal.h" +#include "libc/stdio/lock.h" #include "libc/stdio/stdio.h" #include "libc/stdio/temp.h" #include "libc/mem/alg.h" diff --git a/third_party/awk/lex.c b/third_party/awk/lex.c index 4bfa01d83..92616986e 100644 --- a/third_party/awk/lex.c +++ b/third_party/awk/lex.c @@ -31,7 +31,7 @@ #include "libc/mem/alg.h" #include "libc/mem/mem.h" #include "libc/runtime/runtime.h" -#include "libc/stdio/lock.internal.h" +#include "libc/stdio/lock.h" #include "libc/stdio/rand.h" #include "libc/stdio/stdio.h" #include "libc/stdio/temp.h" diff --git a/third_party/awk/lib.c b/third_party/awk/lib.c index 287619ec8..08cfef0de 100644 --- a/third_party/awk/lib.c +++ b/third_party/awk/lib.c @@ -36,7 +36,7 @@ #include "libc/mem/mem.h" #include "libc/nexgen32e/ffs.h" #include "libc/runtime/runtime.h" -#include "libc/stdio/lock.internal.h" +#include "libc/stdio/lock.h" #include "libc/stdio/rand.h" #include "libc/stdio/stdio.h" #include "libc/stdio/temp.h" diff --git a/third_party/dlmalloc/dlmalloc.mk b/third_party/dlmalloc/dlmalloc.mk index 3a5c2e600..a4e1c34fd 100644 --- a/third_party/dlmalloc/dlmalloc.mk +++ b/third_party/dlmalloc/dlmalloc.mk @@ -34,8 +34,7 @@ THIRD_PARTY_DLMALLOC_A_DIRECTDEPS = \ LIBC_STUBS \ LIBC_SYSV \ LIBC_SYSV_CALLS \ - THIRD_PARTY_COMPILER_RT \ - THIRD_PARTY_NSYNC + THIRD_PARTY_COMPILER_RT THIRD_PARTY_DLMALLOC_A_DEPS := \ $(call uniq,$(foreach x,$(THIRD_PARTY_DLMALLOC_A_DIRECTDEPS),$($(x)))) diff --git a/third_party/dlmalloc/locks.inc b/third_party/dlmalloc/locks.inc index 5084a5d38..e3b722b27 100644 --- a/third_party/dlmalloc/locks.inc +++ b/third_party/dlmalloc/locks.inc @@ -1,6 +1,7 @@ // clang-format off -#include "third_party/nsync/mu.h" -#include "libc/thread/tls.h" +#include "libc/calls/calls.h" +#include "libc/intrin/pthread.h" +#include "libc/nexgen32e/threaded.h" /* --------------------------- Lock preliminaries ------------------------ */ @@ -38,20 +39,233 @@ #define DESTROY_LOCK(l) (0) #define ACQUIRE_MALLOC_GLOBAL_LOCK() #define RELEASE_MALLOC_GLOBAL_LOCK() + #else -#define MLOCK_T nsync_mu -#define ACQUIRE_LOCK(lk) (__threaded && (nsync_mu_lock(lk), 0)) -#define RELEASE_LOCK(lk) (__threaded && (nsync_mu_unlock(lk), 0)) -#define TRY_LOCK(lk) (__threaded ? nsync_mu_trylock(lk) : 1) -#define INITIAL_LOCK(lk) memset(lk, 0, sizeof(*lk)) -#define DESTROY_LOCK(lk) memset(lk, -1, sizeof(*lk)) -#define ACQUIRE_MALLOC_GLOBAL_LOCK() ACQUIRE_LOCK(&malloc_global_mutex); -#define RELEASE_MALLOC_GLOBAL_LOCK() RELEASE_LOCK(&malloc_global_mutex); +#if USE_LOCKS > 1 +/* ----------------------- User-defined locks ------------------------ */ +/* Define your own lock implementation here */ +/* #define INITIAL_LOCK(lk) ... */ +/* #define DESTROY_LOCK(lk) ... */ +/* #define ACQUIRE_LOCK(lk) ... */ +/* #define RELEASE_LOCK(lk) ... */ +/* #define TRY_LOCK(lk) ... */ +/* static MLOCK_T malloc_global_mutex = ... */ + +#define MLOCK_T pthread_mutex_t +#define ACQUIRE_LOCK(lk) (__threaded && pthread_mutex_lock(lk), 0) +#define RELEASE_LOCK(lk) (__threaded && pthread_mutex_unlock(lk), 0) +#define TRY_LOCK(lk) (__threaded ? !pthread_mutex_trylock(lk) : 1) +#define INITIAL_LOCK(lk) pthread_mutex_init(lk, 0) +#define DESTROY_LOCK(lk) pthread_mutex_destroy(lk) + static MLOCK_T malloc_global_mutex; + +#elif USE_SPIN_LOCKS + +/* First, define CAS_LOCK and CLEAR_LOCK on ints */ +/* Note CAS_LOCK defined to return 0 on success */ + +#if defined(__GNUC__)&& (__GNUC__ > 4 || (__GNUC__ == 4 && __GNUC_MINOR__ >= 1)) +#define CAS_LOCK(sl) __sync_lock_test_and_set(sl, 1) +#define CLEAR_LOCK(sl) __sync_lock_release(sl) + +#elif (defined(__GNUC__) && (defined(__i386__) || defined(__x86_64__))) +/* Custom spin locks for older gcc on x86 */ +FORCEINLINE int x86_cas_lock(int *sl) { + int ret; + int val = 1; + int cmp = 0; + __asm__ __volatile__ ("lock; cmpxchgl %1, %2" + : "=a" (ret) + : "r" (val), "m" (*(sl)), "0"(cmp) + : "memory", "cc"); + return ret; +} + +FORCEINLINE void x86_clear_lock(int* sl) { + assert(*sl != 0); + int prev = 0; + int ret; + __asm__ __volatile__ ("lock; xchgl %0, %1" + : "=r" (ret) + : "m" (*(sl)), "0"(prev) + : "memory"); +} + +#define CAS_LOCK(sl) x86_cas_lock(sl) +#define CLEAR_LOCK(sl) x86_clear_lock(sl) + +#else /* Win32 MSC */ +#define CAS_LOCK(sl) interlockedexchange(sl, (LONG)1) +#define CLEAR_LOCK(sl) interlockedexchange (sl, (LONG)0) + +#endif /* ... gcc spins locks ... */ + +#if !defined(USE_RECURSIVE_LOCKS) || USE_RECURSIVE_LOCKS == 0 +/* Plain spin locks use single word (embedded in malloc_states) */ +static dontinline int spin_acquire_lock(int *sl) { + while (*(volatile int *)sl != 0 || CAS_LOCK(sl)) { + sched_yield(); + } + return 0; +} + +#define MLOCK_T int +#define TRY_LOCK(sl) !CAS_LOCK(sl) +#define RELEASE_LOCK(sl) (__threaded && (CLEAR_LOCK(sl), 0)) +#define ACQUIRE_LOCK(sl) (__threaded && CAS_LOCK(sl) ? spin_acquire_lock(sl) : 0) +#define INITIAL_LOCK(sl) (*sl = 0) +#define DESTROY_LOCK(sl) (0) +static MLOCK_T malloc_global_mutex = 0; + +#else /* USE_RECURSIVE_LOCKS */ +/* types for lock owners */ +#ifdef WIN32 +#define THREAD_ID_T DWORD +#define CURRENT_THREAD GetCurrentThreadId() +#define EQ_OWNER(X,Y) ((X) == (Y)) +#else +/* + Note: the following assume that pthread_t is a type that can be + initialized to (casted) zero. If this is not the case, you will need to + somehow redefine these or not use spin locks. +*/ +#define THREAD_ID_T pthread_t +#define CURRENT_THREAD pthread_self() +#define EQ_OWNER(X,Y) pthread_equal(X, Y) #endif +struct malloc_recursive_lock { + int sl; + unsigned int c; + THREAD_ID_T threadid; +}; + +#define MLOCK_T struct malloc_recursive_lock +static MLOCK_T malloc_global_mutex = { 0, 0, (THREAD_ID_T)0}; + +FORCEINLINE void recursive_release_lock(MLOCK_T *lk) { + assert(lk->sl != 0); + if (--lk->c == 0) { + CLEAR_LOCK(&lk->sl); + } +} + +FORCEINLINE int recursive_acquire_lock(MLOCK_T *lk) { + THREAD_ID_T mythreadid = CURRENT_THREAD; + for (;;) { + if (*((volatile int *)(&lk->sl)) == 0) { + if (!CAS_LOCK(&lk->sl)) { + lk->threadid = mythreadid; + lk->c = 1; + return 0; + } + } + else if (EQ_OWNER(lk->threadid, mythreadid)) { + ++lk->c; + return 0; + } + sched_yield(); + } +} + +FORCEINLINE int recursive_try_lock(MLOCK_T *lk) { + THREAD_ID_T mythreadid = CURRENT_THREAD; + if (*((volatile int *)(&lk->sl)) == 0) { + if (!CAS_LOCK(&lk->sl)) { + lk->threadid = mythreadid; + lk->c = 1; + return 1; + } + } + else if (EQ_OWNER(lk->threadid, mythreadid)) { + ++lk->c; + return 1; + } + return 0; +} + +#define RELEASE_LOCK(lk) recursive_release_lock(lk) +#define TRY_LOCK(lk) recursive_try_lock(lk) +#define ACQUIRE_LOCK(lk) recursive_acquire_lock(lk) +#define INITIAL_LOCK(lk) ((lk)->threadid = (THREAD_ID_T)0, (lk)->sl = 0, (lk)->c = 0) +#define DESTROY_LOCK(lk) (0) +#endif /* USE_RECURSIVE_LOCKS */ + +#elif defined(WIN32) /* Win32 critical sections */ +#define MLOCK_T CRITICAL_SECTION +#define ACQUIRE_LOCK(lk) (EnterCriticalSection(lk), 0) +#define RELEASE_LOCK(lk) LeaveCriticalSection(lk) +#define TRY_LOCK(lk) TryEnterCriticalSection(lk) +#define INITIAL_LOCK(lk) (!InitializeCriticalSectionAndSpinCount((lk), 0x80000000|4000)) +#define DESTROY_LOCK(lk) (DeleteCriticalSection(lk), 0) +#define NEED_GLOBAL_LOCK_INIT + +static MLOCK_T malloc_global_mutex; +static volatile LONG malloc_global_mutex_status; + +/* Use spin loop to initialize global lock */ +static void init_malloc_global_mutex() { + for (;;) { + long stat = malloc_global_mutex_status; + if (stat > 0) + return; + /* transition to < 0 while initializing, then to > 0) */ + if (stat == 0 && + interlockedcompareexchange(&malloc_global_mutex_status, (LONG)-1, (LONG)0) == 0) { + InitializeCriticalSection(&malloc_global_mutex); + interlockedexchange(&malloc_global_mutex_status, (LONG)1); + return; + } + SleepEx(0, FALSE); + } +} + +#else /* pthreads-based locks */ +#define MLOCK_T pthread_mutex_t +#define ACQUIRE_LOCK(lk) pthread_mutex_lock(lk) +#define RELEASE_LOCK(lk) pthread_mutex_unlock(lk) +#define TRY_LOCK(lk) (!pthread_mutex_trylock(lk)) +#define INITIAL_LOCK(lk) pthread_init_lock(lk) +#define DESTROY_LOCK(lk) pthread_mutex_destroy(lk) + +#if defined(USE_RECURSIVE_LOCKS) && USE_RECURSIVE_LOCKS != 0 && defined(linux) && !defined(PTHREAD_MUTEX_RECURSIVE) +/* Cope with old-style linux recursive lock initialization by adding */ +/* skipped internal declaration from pthread.h */ +extern int pthread_mutexattr_setkind_np __P ((pthread_mutexattr_t *__attr, + int __kind)); +#define PTHREAD_MUTEX_RECURSIVE PTHREAD_MUTEX_RECURSIVE_NP +#define pthread_mutexattr_settype(x,y) pthread_mutexattr_setkind_np(x,y) +#endif /* USE_RECURSIVE_LOCKS ... */ + +static MLOCK_T malloc_global_mutex = PTHREAD_MUTEX_INITIALIZER; + +static int pthread_init_lock (MLOCK_T *lk) { + pthread_mutexattr_t attr; + if (pthread_mutexattr_init(&attr)) return 1; +#if defined(USE_RECURSIVE_LOCKS) && USE_RECURSIVE_LOCKS != 0 + if (pthread_mutexattr_settype(&attr, PTHREAD_MUTEX_RECURSIVE)) return 1; +#endif + if (pthread_mutex_init(lk, &attr)) return 1; + if (pthread_mutexattr_destroy(&attr)) return 1; + return 0; +} + +#endif /* ... lock types ... */ + +/* Common code for all lock types */ #define USE_LOCK_BIT (2U) +#ifndef ACQUIRE_MALLOC_GLOBAL_LOCK +#define ACQUIRE_MALLOC_GLOBAL_LOCK() ACQUIRE_LOCK(&malloc_global_mutex); +#endif + +#ifndef RELEASE_MALLOC_GLOBAL_LOCK +#define RELEASE_MALLOC_GLOBAL_LOCK() RELEASE_LOCK(&malloc_global_mutex); +#endif + +#endif /* USE_LOCKS */ + struct malloc_chunk { size_t prev_foot; /* Size of previous chunk (if free). */ size_t head; /* Size and inuse bits. */ diff --git a/third_party/gdtoa/lock.c b/third_party/gdtoa/lock.c index 6297114c7..18290c86a 100644 --- a/third_party/gdtoa/lock.c +++ b/third_party/gdtoa/lock.c @@ -16,7 +16,7 @@ │ TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR │ │ PERFORMANCE OF THIS SOFTWARE. │ ╚─────────────────────────────────────────────────────────────────────────────*/ -#include "libc/thread/thread.h" +#include "libc/intrin/pthread.h" #include "third_party/gdtoa/lock.h" static pthread_mutex_t __gdtoa_lock_obj; diff --git a/third_party/gdtoa/lock.h b/third_party/gdtoa/lock.h index 107bc3358..e3b281f0c 100644 --- a/third_party/gdtoa/lock.h +++ b/third_party/gdtoa/lock.h @@ -1,7 +1,7 @@ #ifndef COSMOPOLITAN_THIRD_PARTY_GDTOA_LOCK_H_ #define COSMOPOLITAN_THIRD_PARTY_GDTOA_LOCK_H_ -#include "libc/intrin/nopl.internal.h" -#include "libc/thread/tls.h" +#include "libc/intrin/nopl.h" +#include "libc/nexgen32e/threaded.h" #if !(__ASSEMBLER__ + __LINKER__ + 0) COSMOPOLITAN_C_START_ diff --git a/third_party/libcxx/__threading_support b/third_party/libcxx/__threading_support index b67666d84..d7a0b2114 100644 --- a/third_party/libcxx/__threading_support +++ b/third_party/libcxx/__threading_support @@ -13,7 +13,7 @@ #include "third_party/libcxx/__config" #include "third_party/libcxx/chrono" #include "third_party/libcxx/iosfwd" -#include "libc/thread/thread2.h" +#include "libc/intrin/pthread2.h" #include "third_party/libcxx/errno.h" #ifndef _LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER @@ -25,7 +25,7 @@ #elif !defined(_LIBCPP_HAS_NO_THREADS) #if defined(_LIBCPP_HAS_THREAD_API_PTHREAD) -#include "libc/thread/thread.h" +#include "libc/intrin/pthread.h" #include "libc/calls/calls.h" #include "libc/calls/struct/sched_param.h" #include "libc/sysv/consts/sched.h" diff --git a/third_party/lua/liolib.c b/third_party/lua/liolib.c index 6eced3349..89d6fa9ea 100644 --- a/third_party/lua/liolib.c +++ b/third_party/lua/liolib.c @@ -30,7 +30,7 @@ #include "libc/calls/calls.h" #include "libc/calls/weirdtypes.h" #include "libc/errno.h" -#include "libc/stdio/lock.internal.h" +#include "libc/stdio/lock.h" #include "libc/stdio/stdio.h" #include "libc/stdio/temp.h" #include "libc/str/str.h" diff --git a/third_party/lua/lrepl.c b/third_party/lua/lrepl.c index 1a47d135c..d99a5998c 100644 --- a/third_party/lua/lrepl.c +++ b/third_party/lua/lrepl.c @@ -31,7 +31,7 @@ #include "libc/calls/struct/sigaction.h" #include "libc/errno.h" #include "libc/intrin/nomultics.internal.h" -#include "libc/thread/thread.h" +#include "libc/intrin/pthread.h" #include "libc/log/check.h" #include "libc/macros.internal.h" #include "libc/mem/mem.h" diff --git a/third_party/lua/lrepl.h b/third_party/lua/lrepl.h index 199f696a2..f61240bcc 100644 --- a/third_party/lua/lrepl.h +++ b/third_party/lua/lrepl.h @@ -1,6 +1,6 @@ #ifndef COSMOPOLITAN_THIRD_PARTY_LUA_LREPL_H_ #define COSMOPOLITAN_THIRD_PARTY_LUA_LREPL_H_ -#include "libc/intrin/nopl.internal.h" +#include "libc/intrin/nopl.h" #include "third_party/linenoise/linenoise.h" #include "third_party/lua/lauxlib.h" #if !(__ASSEMBLER__ + __LINKER__ + 0) diff --git a/third_party/nsync/LICENSE.txt b/third_party/nsync/LICENSE.txt deleted file mode 100644 index d64569567..000000000 --- a/third_party/nsync/LICENSE.txt +++ /dev/null @@ -1,202 +0,0 @@ - - Apache License - Version 2.0, January 2004 - http://www.apache.org/licenses/ - - TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION - - 1. Definitions. - - "License" shall mean the terms and conditions for use, reproduction, - and distribution as defined by Sections 1 through 9 of this document. - - "Licensor" shall mean the copyright owner or entity authorized by - the copyright owner that is granting the License. - - "Legal Entity" shall mean the union of the acting entity and all - other entities that control, are controlled by, or are under common - control with that entity. For the purposes of this definition, - "control" means (i) the power, direct or indirect, to cause the - direction or management of such entity, whether by contract or - otherwise, or (ii) ownership of fifty percent (50%) or more of the - outstanding shares, or (iii) beneficial ownership of such entity. - - "You" (or "Your") shall mean an individual or Legal Entity - exercising permissions granted by this License. - - "Source" form shall mean the preferred form for making modifications, - including but not limited to software source code, documentation - source, and configuration files. - - "Object" form shall mean any form resulting from mechanical - transformation or translation of a Source form, including but - not limited to compiled object code, generated documentation, - and conversions to other media types. - - "Work" shall mean the work of authorship, whether in Source or - Object form, made available under the License, as indicated by a - copyright notice that is included in or attached to the work - (an example is provided in the Appendix below). - - "Derivative Works" shall mean any work, whether in Source or Object - form, that is based on (or derived from) the Work and for which the - editorial revisions, annotations, elaborations, or other modifications - represent, as a whole, an original work of authorship. For the purposes - of this License, Derivative Works shall not include works that remain - separable from, or merely link (or bind by name) to the interfaces of, - the Work and Derivative Works thereof. - - "Contribution" shall mean any work of authorship, including - the original version of the Work and any modifications or additions - to that Work or Derivative Works thereof, that is intentionally - submitted to Licensor for inclusion in the Work by the copyright owner - or by an individual or Legal Entity authorized to submit on behalf of - the copyright owner. For the purposes of this definition, "submitted" - means any form of electronic, verbal, or written communication sent - to the Licensor or its representatives, including but not limited to - communication on electronic mailing lists, source code control systems, - and issue tracking systems that are managed by, or on behalf of, the - Licensor for the purpose of discussing and improving the Work, but - excluding communication that is conspicuously marked or otherwise - designated in writing by the copyright owner as "Not a Contribution." - - "Contributor" shall mean Licensor and any individual or Legal Entity - on behalf of whom a Contribution has been received by Licensor and - subsequently incorporated within the Work. - - 2. Grant of Copyright License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - copyright license to reproduce, prepare Derivative Works of, - publicly display, publicly perform, sublicense, and distribute the - Work and such Derivative Works in Source or Object form. - - 3. Grant of Patent License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - (except as stated in this section) patent license to make, have made, - use, offer to sell, sell, import, and otherwise transfer the Work, - where such license applies only to those patent claims licensable - by such Contributor that are necessarily infringed by their - Contribution(s) alone or by combination of their Contribution(s) - with the Work to which such Contribution(s) was submitted. If You - institute patent litigation against any entity (including a - cross-claim or counterclaim in a lawsuit) alleging that the Work - or a Contribution incorporated within the Work constitutes direct - or contributory patent infringement, then any patent licenses - granted to You under this License for that Work shall terminate - as of the date such litigation is filed. - - 4. Redistribution. You may reproduce and distribute copies of the - Work or Derivative Works thereof in any medium, with or without - modifications, and in Source or Object form, provided that You - meet the following conditions: - - (a) You must give any other recipients of the Work or - Derivative Works a copy of this License; and - - (b) You must cause any modified files to carry prominent notices - stating that You changed the files; and - - (c) You must retain, in the Source form of any Derivative Works - that You distribute, all copyright, patent, trademark, and - attribution notices from the Source form of the Work, - excluding those notices that do not pertain to any part of - the Derivative Works; and - - (d) If the Work includes a "NOTICE" text file as part of its - distribution, then any Derivative Works that You distribute must - include a readable copy of the attribution notices contained - within such NOTICE file, excluding those notices that do not - pertain to any part of the Derivative Works, in at least one - of the following places: within a NOTICE text file distributed - as part of the Derivative Works; within the Source form or - documentation, if provided along with the Derivative Works; or, - within a display generated by the Derivative Works, if and - wherever such third-party notices normally appear. The contents - of the NOTICE file are for informational purposes only and - do not modify the License. You may add Your own attribution - notices within Derivative Works that You distribute, alongside - or as an addendum to the NOTICE text from the Work, provided - that such additional attribution notices cannot be construed - as modifying the License. - - You may add Your own copyright statement to Your modifications and - may provide additional or different license terms and conditions - for use, reproduction, or distribution of Your modifications, or - for any such Derivative Works as a whole, provided Your use, - reproduction, and distribution of the Work otherwise complies with - the conditions stated in this License. - - 5. Submission of Contributions. Unless You explicitly state otherwise, - any Contribution intentionally submitted for inclusion in the Work - by You to the Licensor shall be under the terms and conditions of - this License, without any additional terms or conditions. - Notwithstanding the above, nothing herein shall supersede or modify - the terms of any separate license agreement you may have executed - with Licensor regarding such Contributions. - - 6. Trademarks. This License does not grant permission to use the trade - names, trademarks, service marks, or product names of the Licensor, - except as required for reasonable and customary use in describing the - origin of the Work and reproducing the content of the NOTICE file. - - 7. Disclaimer of Warranty. Unless required by applicable law or - agreed to in writing, Licensor provides the Work (and each - Contributor provides its Contributions) on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or - implied, including, without limitation, any warranties or conditions - of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A - PARTICULAR PURPOSE. You are solely responsible for determining the - appropriateness of using or redistributing the Work and assume any - risks associated with Your exercise of permissions under this License. - - 8. Limitation of Liability. In no event and under no legal theory, - whether in tort (including negligence), contract, or otherwise, - unless required by applicable law (such as deliberate and grossly - negligent acts) or agreed to in writing, shall any Contributor be - liable to You for damages, including any direct, indirect, special, - incidental, or consequential damages of any character arising as a - result of this License or out of the use or inability to use the - Work (including but not limited to damages for loss of goodwill, - work stoppage, computer failure or malfunction, or any and all - other commercial damages or losses), even if such Contributor - has been advised of the possibility of such damages. - - 9. Accepting Warranty or Additional Liability. While redistributing - the Work or Derivative Works thereof, You may choose to offer, - and charge a fee for, acceptance of support, warranty, indemnity, - or other liability obligations and/or rights consistent with this - License. However, in accepting such obligations, You may act only - on Your own behalf and on Your sole responsibility, not on behalf - of any other Contributor, and only if You agree to indemnify, - defend, and hold each Contributor harmless for any liability - incurred by, or claims asserted against, such Contributor by reason - of your accepting any such warranty or additional liability. - - END OF TERMS AND CONDITIONS - - APPENDIX: How to apply the Apache License to your work. - - To apply the Apache License to your work, attach the following - boilerplate notice, with the fields enclosed by brackets "[]" - replaced with your own identifying information. (Don't include - the brackets!) The text should be enclosed in the appropriate - comment syntax for the file format. We also recommend that a - file or class name and description of purpose be included on the - same "printed page" as the copyright notice for easier - identification within third-party archives. - - Copyright [yyyy] [name of copyright owner] - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. diff --git a/third_party/nsync/README.md b/third_party/nsync/README.md deleted file mode 100644 index 12dadf03b..000000000 --- a/third_party/nsync/README.md +++ /dev/null @@ -1,21 +0,0 @@ -# *NSYNC - -The `THIRD_PARTY_NSYNC` and `LIBC_THREAD` packages include source code -from *NSYNC. Here's the latest upstream synchronization point: - - git@github.com:google/nsync - ac5489682760393fe21bd2a8e038b528442412a7 (1.25.0) - Author: Mike Burrows - Date: Wed Jun 1 16:47:52 2022 -0700 - -NSYNC uses the Apache 2.0 license. We made the following local changes: - - - Write custom `nsync_malloc_()` so `malloc()` can use *NSYNC. - - - Rewrite `futex()` wrapper to support old Linux kernels and OpenBSD. - - - Normalize sources to Cosmopolitan style conventions; *NSYNC upstream - supports dozens of compilers and operating systems, at compile-time. - Since Cosmo solves portability at runtime instead, most of the build - config toil has been removed, in order to help the NSYNC source code - be more readable and hackable. diff --git a/third_party/nsync/atomic.h b/third_party/nsync/atomic.h deleted file mode 100644 index eeaae199a..000000000 --- a/third_party/nsync/atomic.h +++ /dev/null @@ -1,15 +0,0 @@ -#ifndef NSYNC_ATOMIC_H_ -#define NSYNC_ATOMIC_H_ -#if !(__ASSEMBLER__ + __LINKER__ + 0) -COSMOPOLITAN_C_START_ - -typedef uint32_t nsync_atomic_uint32_; - -#define NSYNC_ATOMIC_UINT32_INIT_ 0 -#define NSYNC_ATOMIC_UINT32_LOAD_(p) (*(p)) -#define NSYNC_ATOMIC_UINT32_STORE_(p, v) (*(p) = (v)) -#define NSYNC_ATOMIC_UINT32_PTR_(p) (p) - -COSMOPOLITAN_C_END_ -#endif /* !(__ASSEMBLER__ + __LINKER__ + 0) */ -#endif /* NSYNC_ATOMIC_H_ */ diff --git a/third_party/nsync/atomic.internal.h b/third_party/nsync/atomic.internal.h deleted file mode 100644 index 9aa4f87df..000000000 --- a/third_party/nsync/atomic.internal.h +++ /dev/null @@ -1,113 +0,0 @@ -#ifndef NSYNC_ATOMIC_INTERNAL_H_ -#define NSYNC_ATOMIC_INTERNAL_H_ -#include "libc/intrin/atomic.h" -#include "third_party/nsync/atomic.h" -#if !(__ASSEMBLER__ + __LINKER__ + 0) -COSMOPOLITAN_C_START_ - -/* Atomic operations on nsync_atomic_uint32_ quantities - CAS, load, and store. - - Normally, these are used only on nsync_atomic_uint32_ values, but on - Linux they may be invoked on int values, because futexes operate on - int values. A compile-time check in the futex code ensures that both - int and nsync_atomic_uint32_ are 32 bits. - - Memory barriers: - - Operations with the suffixes _ACQ and _RELACQ ensure that the - operation appears to complete before other memory operations - subsequently performed by the same thread, as seen by other - threads. (In the case of ATM_CAS_ACQ, this applies only if - the operation returns a non-zero value.) - - Operations with the suffixes _REL and _RELACQ ensure that the - operation appears to complete after other memory operations - previously performed by the same thread, as seen by other - threads. (In the case of ATM_CAS_REL, this applies only if - the operation returns a non-zero value.) - - // Atomically, - // int ATM_CAS (nsync_atomic_uint32_ *p, - // uint32_t old_value, uint32_t new_value) { - // if (*p == old_value) { - // *p = new_value; - // return (some-non-zero-value); - // } else { - // return (0); - // } - // } - // *_ACQ, *_REL, *_RELACQ variants are available, - // with the barrier semantics described above. - int ATM_CAS (nsync_atomic_uint32_ *p, uint32_t old_value, - uint32_t new_value); - - // Atomically, - // uint32_t ATM_LOAD (nsync_atomic_uint32_ *p) { return (*p); } - // A *_ACQ variant is available, - // with the barrier semantics described above. - uint32_t ATM_LOAD (nsync_atomic_uint32_ *p); - - // Atomically, - // void ATM_STORE (nsync_atomic_uint32_ *p, uint32_t value) { - // *p = value; - // } - // A *_REL variant is available, - // with the barrier semantics described above. - void ATM_STORE (nsync_atomic_uint32_ *p, uint32_t value); - */ - -static inline int atm_cas_nomb_u32_(nsync_atomic_uint32_ *p, uint32_t o, - uint32_t n) { - return atomic_compare_exchange_strong_explicit(NSYNC_ATOMIC_UINT32_PTR_(p), - &o, n, memory_order_relaxed, - memory_order_relaxed); -} - -static inline int atm_cas_acq_u32_(nsync_atomic_uint32_ *p, uint32_t o, - uint32_t n) { - return atomic_compare_exchange_strong_explicit(NSYNC_ATOMIC_UINT32_PTR_(p), - &o, n, memory_order_acquire, - memory_order_relaxed); -} - -static inline int atm_cas_rel_u32_(nsync_atomic_uint32_ *p, uint32_t o, - uint32_t n) { - return atomic_compare_exchange_strong_explicit(NSYNC_ATOMIC_UINT32_PTR_(p), - &o, n, memory_order_release, - memory_order_relaxed); -} - -static inline int atm_cas_relacq_u32_(nsync_atomic_uint32_ *p, uint32_t o, - uint32_t n) { - return atomic_compare_exchange_strong_explicit(NSYNC_ATOMIC_UINT32_PTR_(p), - &o, n, memory_order_acq_rel, - memory_order_relaxed); -} - -#define ATM_CAS_HELPER_(barrier, p, o, n) \ - (atm_cas_##barrier##_u32_((p), (o), (n))) - -#define ATM_CAS(p, o, n) ATM_CAS_HELPER_(nomb, (p), (o), (n)) -#define ATM_CAS_ACQ(p, o, n) ATM_CAS_HELPER_(acq, (p), (o), (n)) -#define ATM_CAS_REL(p, o, n) ATM_CAS_HELPER_(rel, (p), (o), (n)) -#define ATM_CAS_RELACQ(p, o, n) ATM_CAS_HELPER_(relacq, (p), (o), (n)) - -/* Need a cast to remove "const" from some uses. */ -#define ATM_LOAD(p) \ - (atomic_load_explicit((nsync_atomic_uint32_ *)NSYNC_ATOMIC_UINT32_PTR_(p), \ - memory_order_relaxed)) -#define ATM_LOAD_ACQ(p) \ - (atomic_load_explicit((nsync_atomic_uint32_ *)NSYNC_ATOMIC_UINT32_PTR_(p), \ - memory_order_acquire)) - -#define ATM_STORE(p, v) \ - (atomic_store_explicit(NSYNC_ATOMIC_UINT32_PTR_(p), (v), \ - memory_order_relaxed)) -#define ATM_STORE_REL(p, v) \ - (atomic_store_explicit(NSYNC_ATOMIC_UINT32_PTR_(p), (v), \ - memory_order_release)) - -COSMOPOLITAN_C_END_ -#endif /* !(__ASSEMBLER__ + __LINKER__ + 0) */ -#endif /* NSYNC_ATOMIC_INTERNAL_H_ */ diff --git a/third_party/nsync/common.c b/third_party/nsync/common.c deleted file mode 100644 index c722fb967..000000000 --- a/third_party/nsync/common.c +++ /dev/null @@ -1,246 +0,0 @@ -/*-*- mode:c;indent-tabs-mode:t;c-basic-offset:8;tab-width:8;coding:utf-8 -*-│ -│vi: set et ft=c ts=8 tw=8 fenc=utf-8 :vi│ -╞══════════════════════════════════════════════════════════════════════════════╡ -│ Copyright 2016 Google Inc. │ -│ │ -│ Licensed under the Apache License, Version 2.0 (the "License"); │ -│ you may not use this file except in compliance with the License. │ -│ You may obtain a copy of the License at │ -│ │ -│ http://www.apache.org/licenses/LICENSE-2.0 │ -│ │ -│ Unless required by applicable law or agreed to in writing, software │ -│ distributed under the License is distributed on an "AS IS" BASIS, │ -│ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. │ -│ See the License for the specific language governing permissions and │ -│ limitations under the License. │ -╚─────────────────────────────────────────────────────────────────────────────*/ -#include "libc/mem/mem.h" -#include "libc/runtime/runtime.h" -#include "libc/thread/thread.h" -#include "third_party/nsync/atomic.h" -#include "third_party/nsync/atomic.internal.h" -#include "third_party/nsync/common.internal.h" -#include "third_party/nsync/dll.h" -#include "third_party/nsync/malloc.internal.h" -#include "third_party/nsync/mu_semaphore.h" -#include "third_party/nsync/races.internal.h" -#include "third_party/nsync/wait_s.internal.h" - -asm(".ident\t\"\\n\\n\ -*NSYNC (Apache 2.0)\\n\ -Copyright 2016 Google, Inc.\\n\ -https://github.com/google/nsync\""); -// clang-format off - -/* This package provides a mutex nsync_mu and a Mesa-style condition - * variable nsync_cv. */ - -/* Implementation notes - - The implementations of nsync_mu and nsync_cv both use spinlocks to protect - their waiter queues. The spinlocks are implemented with atomic operations - and a delay loop found below. They could use pthread_mutex_t, but I wished - to have an implementation independent of pthread mutexes and condition - variables. - - nsync_mu and nsync_cv use the same type of doubly-linked list of waiters - (see waiter.c). This allows waiters to be transferred from the cv queue to - the mu queue when a thread is logically woken from the cv but would - immediately go to sleep on the mu. See the wake_waiters() call. - - In mu, the "designated waker" is a thread that was waiting on mu, has been - woken up, but as yet has neither acquired nor gone back to waiting. The - presence of such a thread is indicated by the MU_DESIG_WAKER bit in the mu - word. This bit allows the nsync_mu_unlock() code to avoid waking a second - waiter when there's already one that will wake the next thread when the time - comes. This speeds things up when the lock is heavily contended, and the - critical sections are small. - - The weasel words "with high probability" in the specification of - nsync_mu_trylock() and nsync_mu_rtrylock() prevent clients from believing - that they can determine with certainty whether another thread has given up a - lock yet. This, together with the requirement that a thread that acquired a - mutex must release it (rather than it being released by another thread), - prohibits clients from using mu as a sort of semaphore. The intent is that - it be used only for traditional mutual exclusion, and that clients that need - a semaphore should use one. This leaves room for certain future - optimizations, and make it easier to apply detection of potential races via - candidate lock-set algorithms, should that ever be desired. - - The nsync_mu_wait_with_deadline() and nsync_mu_wait_with_deadline() calls use an - absolute rather than a relative timeout. This is less error prone, as - described in the comment on nsync_cv_wait_with_deadline(). Alas, relative - timeouts are seductive in trivial examples (such as tests). These are the - first things that people try, so they are likely to be requested. If enough - people complain we could give them that particular piece of rope. - - Excessive evaluations of the same wait condition are avoided by maintaining - waiter.same_condition as a doubly-linked list of waiters with the same - non-NULL wait condition that are also adjacent in the waiter list. This does - well even with large numbers of threads if there is at most one - wait condition that can be false at any given time (such as in a - producer/consumer queue, which cannot be both empty and full - simultaneously). One could imagine a queueing mechanism that would - guarantee to evaluate each condition at most once per wakeup, but that would - be substantially more complex, and would still degrade if the number of - distinct wakeup conditions were high. So clients are advised to resort to - condition variables if they have many distinct wakeup conditions. */ - -/* Used in spinloops to delay resumption of the loop. - Usage: - unsigned attempts = 0; - while (try_something) { - attempts = nsync_spin_delay_ (attempts); - } */ -unsigned nsync_spin_delay_ (unsigned attempts) { - if (attempts < 7) { - volatile int i; - for (i = 0; i != 1 << attempts; i++) { - } - attempts++; - } else { - nsync_yield_ (); - } - return (attempts); -} - -/* Spin until (*w & test) == 0, then atomically perform *w = ((*w | set) & - ~clear), perform an acquire barrier, and return the previous value of *w. - */ -uint32_t nsync_spin_test_and_set_ (nsync_atomic_uint32_ *w, uint32_t test, - uint32_t set, uint32_t clear) { - unsigned attempts = 0; /* CV_SPINLOCK retry count */ - uint32_t old = ATM_LOAD (w); - while ((old & test) != 0 || !ATM_CAS_ACQ (w, old, (old | set) & ~clear)) { - attempts = nsync_spin_delay_ (attempts); - old = ATM_LOAD (w); - } - return (old); -} - -/* ====================================================================================== */ - -struct nsync_waiter_s *nsync_dll_nsync_waiter_ (nsync_dll_element_ *e) { - struct nsync_waiter_s *nw = (struct nsync_waiter_s *) e->container; - ASSERT (nw->tag == NSYNC_WAITER_TAG); - ASSERT (e == &nw->q); - return (nw); -} -waiter *nsync_dll_waiter_ (nsync_dll_element_ *e) { - struct nsync_waiter_s *nw = DLL_NSYNC_WAITER (e); - waiter *w = CONTAINER (waiter, nw, nw); - ASSERT ((nw->flags & NSYNC_WAITER_FLAG_MUCV) != 0); - ASSERT (w->tag == WAITER_TAG); - ASSERT (e == &w->nw.q); - return (w); -} - -waiter *nsync_dll_waiter_samecond_ (nsync_dll_element_ *e) { - waiter *w = (waiter *) e->container; - ASSERT (w->tag == WAITER_TAG); - ASSERT (e == &w->same_condition); - return (w); -} - -/* -------------------------------- */ - -static nsync_dll_list_ free_waiters = NULL; - -/* free_waiters points to a doubly-linked list of free waiter structs. */ -static nsync_atomic_uint32_ free_waiters_mu; /* spinlock; protects free_waiters */ - -static _Thread_local waiter *waiter_for_thread; - -static void waiter_destroy (void *v) { - waiter *w = (waiter *) v; - /* Reset waiter_for_thread in case another thread-local variable reuses - the waiter in its destructor while the waiter is taken by the other - thread from free_waiters. This can happen as the destruction order - of thread-local variables can be arbitrary in some platform e.g. - POSIX. */ - waiter_for_thread = NULL; - IGNORE_RACES_START (); - ASSERT ((w->flags & (WAITER_RESERVED|WAITER_IN_USE)) == WAITER_RESERVED); - w->flags &= ~WAITER_RESERVED; - nsync_spin_test_and_set_ (&free_waiters_mu, 1, 1, 0); - free_waiters = nsync_dll_make_first_in_list_ (free_waiters, &w->nw.q); - ATM_STORE_REL (&free_waiters_mu, 0); /* release store */ - IGNORE_RACES_END (); -} - -/* Return a pointer to an unused waiter struct. - Ensures that the enclosed timer is stopped and its channel drained. */ -waiter *nsync_waiter_new_ (void) { - nsync_dll_element_ *q; - waiter *tw; - waiter *w; - tw = waiter_for_thread; - w = tw; - if (w == NULL || (w->flags & (WAITER_RESERVED|WAITER_IN_USE)) != WAITER_RESERVED) { - w = NULL; - nsync_spin_test_and_set_ (&free_waiters_mu, 1, 1, 0); - q = nsync_dll_first_ (free_waiters); - if (q != NULL) { /* If free list is non-empty, dequeue an item. */ - free_waiters = nsync_dll_remove_ (free_waiters, q); - w = DLL_WAITER (q); - } - ATM_STORE_REL (&free_waiters_mu, 0); /* release store */ - if (w == NULL) { /* If free list was empty, allocate an item. */ - w = (waiter *) nsync_malloc_ (sizeof (*w)); - w->tag = WAITER_TAG; - w->nw.tag = NSYNC_WAITER_TAG; - nsync_mu_semaphore_init (&w->sem); - w->nw.sem = &w->sem; - nsync_dll_init_ (&w->nw.q, &w->nw); - NSYNC_ATOMIC_UINT32_STORE_ (&w->nw.waiting, 0); - w->nw.flags = NSYNC_WAITER_FLAG_MUCV; - ATM_STORE (&w->remove_count, 0); - nsync_dll_init_ (&w->same_condition, w); - w->flags = 0; - } - if (tw == NULL) { - w->flags |= WAITER_RESERVED; - nsync_set_per_thread_waiter_ (w, &waiter_destroy); - waiter_for_thread = w; - } - } - w->flags |= WAITER_IN_USE; - return (w); -} - -/* Return an unused waiter struct *w to the free pool. */ -void nsync_waiter_free_ (waiter *w) { - ASSERT ((w->flags & WAITER_IN_USE) != 0); - w->flags &= ~WAITER_IN_USE; - if ((w->flags & WAITER_RESERVED) == 0) { - nsync_spin_test_and_set_ (&free_waiters_mu, 1, 1, 0); - free_waiters = nsync_dll_make_first_in_list_ (free_waiters, &w->nw.q); - ATM_STORE_REL (&free_waiters_mu, 0); /* release store */ - } -} - -/* ====================================================================================== */ - -/* writer_type points to a lock_type that describes how to manipulate a mu for a writer. */ -static lock_type Xwriter_type = { - MU_WZERO_TO_ACQUIRE, - MU_WADD_TO_ACQUIRE, - MU_WHELD_IF_NON_ZERO, - MU_WSET_WHEN_WAITING, - MU_WCLEAR_ON_ACQUIRE, - MU_WCLEAR_ON_UNCONTENDED_RELEASE -}; -lock_type *nsync_writer_type_ = &Xwriter_type; - - -/* reader_type points to a lock_type that describes how to manipulate a mu for a reader. */ -static lock_type Xreader_type = { - MU_RZERO_TO_ACQUIRE, - MU_RADD_TO_ACQUIRE, - MU_RHELD_IF_NON_ZERO, - MU_RSET_WHEN_WAITING, - MU_RCLEAR_ON_ACQUIRE, - MU_RCLEAR_ON_UNCONTENDED_RELEASE -}; -lock_type *nsync_reader_type_ = &Xreader_type; diff --git a/third_party/nsync/common.internal.h b/third_party/nsync/common.internal.h deleted file mode 100644 index 91553e49c..000000000 --- a/third_party/nsync/common.internal.h +++ /dev/null @@ -1,290 +0,0 @@ -#ifndef NSYNC_COMMON_H_ -#define NSYNC_COMMON_H_ -#include "third_party/nsync/atomic.h" -#include "third_party/nsync/atomic.internal.h" -#include "third_party/nsync/cv.h" -#include "third_party/nsync/dll.h" -#include "third_party/nsync/mu.h" -#include "third_party/nsync/mu_semaphore.h" -#include "third_party/nsync/note.h" -#include "third_party/nsync/time.h" -#include "third_party/nsync/wait_s.internal.h" -#if !(__ASSEMBLER__ + __LINKER__ + 0) -COSMOPOLITAN_C_START_ - -#ifndef NSYNC_DEBUG -#define NSYNC_DEBUG 0 -#endif - -/* Yield the CPU. Platform specific. */ -void nsync_yield_(void); - -/* Retrieve the per-thread cache of the waiter object. Platform specific. */ -void *nsync_per_thread_waiter_(void (*dest)(void *)); - -/* Set the per-thread cache of the waiter object. Platform specific. */ -void nsync_set_per_thread_waiter_(void *v, void (*dest)(void *)); - -/* Used in spinloops to delay resumption of the loop. - Usage: - unsigned attempts = 0; - while (try_something) { - attempts = nsync_spin_delay_ (attempts); - } */ -unsigned nsync_spin_delay_(unsigned attempts); - -/* Spin until (*w & test) == 0, then atomically perform *w = ((*w | set) & - ~clear), perform an acquire barrier, and return the previous value of *w. - */ -uint32_t nsync_spin_test_and_set_(nsync_atomic_uint32_ *w, uint32_t test, - uint32_t set, uint32_t clear); - -/* Abort after printing the nul-temrinated string s[]. */ -void nsync_panic_(const char *s); - -/* ---------- */ - -#define MIN_(a_, b_) ((a_) < (b_) ? (a_) : (b_)) -#define MAX_(a_, b_) ((a_) > (b_) ? (a_) : (b_)) - -/* ---------- */ - -/* Fields in nsync_mu.word. - - - At least one of the MU_WLOCK or MU_RLOCK_FIELD fields must be zero. - - MU_WLOCK indicates that a write lock is held. - - MU_RLOCK_FIELD is a count of readers with read locks. - - - MU_SPINLOCK represents a spinlock that must be held when manipulating the - waiter queue. - - - MU_DESIG_WAKER indicates that a former waiter has been woken, but has - neither acquired the lock nor gone back to sleep. Legal to fail to set it; - illegal to set it when no such waiter exists. - - - MU_WAITING indicates whether the waiter queue is non-empty. - The following bits should be zero if MU_WAITING is zero. - - MU_CONDITION indicates that some waiter may have an associated condition - (from nsync_mu_wait, etc.). Legal to set it with no such waiter exists, - but illegal to fail to set it with such a waiter. - - MU_WRITER_WAITING indicates that a reader that has not yet blocked - at least once should not acquire in order not to starve waiting writers. - It set when a writer blocks or a reader is woken with a writer waiting. - It is reset when a writer acquires, but set again when that writer - releases if it wakes readers and there is a waiting writer. - - MU_LONG_WAIT indicates that a waiter has been woken many times but - repeatedly failed to acquire when competing for the lock. This is used - only to prevent long-term starvation by writers. The thread that sets it - clears it when if acquires. - - MU_ALL_FALSE indicates that a complete scan of the waiter list found no - waiters with true conditions, and the lock has not been acquired by a - writer since then. This allows a reader lock to be released without - testing conditions again. It is legal to fail to set this, but illegal - to set it inappropriately. - */ -#define MU_WLOCK ((uint32_t)(1 << 0)) /* writer lock is held. */ -#define MU_SPINLOCK \ - ((uint32_t)(1 << 1)) /* spinlock is held (protects waiters). */ -#define MU_WAITING ((uint32_t)(1 << 2)) /* waiter list is non-empty. */ -#define MU_DESIG_WAKER \ - ((uint32_t)(1 << 3)) /* a former waiter awoke, and hasn't yet acquired or \ - slept anew */ -#define MU_CONDITION \ - ((uint32_t)(1 << 4)) /* the wait list contains some conditional waiters. */ -#define MU_WRITER_WAITING ((uint32_t)(1 << 5)) /* there is a writer waiting */ -#define MU_LONG_WAIT \ - ((uint32_t)(1 << 6)) /* the waiter at the head of the queue has been waiting \ - a long time */ -#define MU_ALL_FALSE \ - ((uint32_t)(1 << 7)) /* all waiter conditions are false \ - */ -#define MU_RLOCK \ - ((uint32_t)( \ - 1 << 8)) /* low-order bit of reader count, which uses rest of word */ - -/* The constants below are derived from those above. */ -#define MU_RLOCK_FIELD \ - (~(uint32_t)(MU_RLOCK - 1)) /* mask of reader count field */ - -#define MU_ANY_LOCK (MU_WLOCK | MU_RLOCK_FIELD) /* mask for any lock held */ - -#define MU_WZERO_TO_ACQUIRE \ - (MU_ANY_LOCK | MU_LONG_WAIT) /* bits to be zero to acquire write lock */ -#define MU_WADD_TO_ACQUIRE (MU_WLOCK) /* add to acquire a write lock */ -#define MU_WHELD_IF_NON_ZERO \ - (MU_WLOCK) /* if any of these bits are set, write lock is held */ -#define MU_WSET_WHEN_WAITING \ - (MU_WAITING | MU_WRITER_WAITING) /* a writer is waiting */ -#define MU_WCLEAR_ON_ACQUIRE \ - (MU_WRITER_WAITING) /* clear MU_WRITER_WAITING when a writer acquires */ -#define MU_WCLEAR_ON_UNCONTENDED_RELEASE \ - (MU_ALL_FALSE) /* clear if a writer releases w/o waking */ - -/* bits to be zero to acquire read lock */ -#define MU_RZERO_TO_ACQUIRE (MU_WLOCK | MU_WRITER_WAITING | MU_LONG_WAIT) -#define MU_RADD_TO_ACQUIRE (MU_RLOCK) /* add to acquire a read lock */ -#define MU_RHELD_IF_NON_ZERO \ - (MU_RLOCK_FIELD) /* if any of these bits are set, read lock is held */ -#define MU_RSET_WHEN_WAITING \ - (MU_WAITING) /* indicate that some thread is waiting */ -#define MU_RCLEAR_ON_ACQUIRE \ - ((uint32_t)0) /* nothing to clear when a read acquires */ -#define MU_RCLEAR_ON_UNCONTENDED_RELEASE \ - ((uint32_t)0) /* nothing to clear when a read releases */ - -/* A lock_type holds the values needed to manipulate a mu in some mode (read or - write). This allows some of the code to be generic, and parameterized by - the lock type. */ -typedef struct lock_type_s { - uint32_t zero_to_acquire; /* bits that must be zero to acquire */ - uint32_t add_to_acquire; /* constant to add to acquire */ - uint32_t - held_if_non_zero; /* if any of these bits are set, the lock is held */ - uint32_t set_when_waiting; /* set when thread waits */ - uint32_t clear_on_acquire; /* clear when thread acquires */ - uint32_t clear_on_uncontended_release; /* clear when thread releases without - waking */ -} lock_type; - -/* writer_type points to a lock_type that describes how to manipulate a mu for a - * writer. */ -extern lock_type *nsync_writer_type_; - -/* reader_type points to a lock_type that describes how to manipulate a mu for a - * reader. */ -extern lock_type *nsync_reader_type_; - -/* ---------- */ - -/* Bits in nsync_cv.word */ - -#define CV_SPINLOCK ((uint32_t)(1 << 0)) /* protects waiters */ -#define CV_NON_EMPTY ((uint32_t)(1 << 1)) /* waiters list is non-empty */ - -/* ---------- */ - -/* Hold a pair of condition function and its argument. */ -struct wait_condition_s { - int (*f)(const void *v); - const void *v; - int (*eq)(const void *a, const void *b); -}; - -/* Return whether wait conditions *a_ and *b_ are equal and non-null. */ -#define WAIT_CONDITION_EQ(a_, b_) \ - ((a_)->f != NULL && (a_)->f == (b_)->f && \ - ((a_)->v == (b_)->v || \ - ((a_)->eq != NULL && (*(a_)->eq)((a_)->v, (b_)->v)))) - -/* If a waiter has waited this many times, it may set the MU_LONG_WAIT bit. */ -#define LONG_WAIT_THRESHOLD 30 - -/* ---------- */ - -#define NOTIFIED_TIME(n_) \ - (ATM_LOAD_ACQ(&(n_)->notified) != 0 ? nsync_time_zero \ - : (n_)->expiry_time_valid ? (n_)->expiry_time \ - : nsync_time_no_deadline) - -/* A waiter represents a single waiter on a cv or a mu. - - To wait: - Allocate a waiter struct *w with new_waiter(), set w.waiting=1, and - w.cv_mu=nil or to the associated mu if waiting on a condition variable, then - queue w.nsync_dll on some queue, and then wait using: - while (ATM_LOAD_ACQ (&w.waiting) != 0) { nsync_mu_semaphore_p (&w.sem); } - Return *w to the freepool by calling free_waiter (w). - - To wakeup: - Remove *w from the relevant queue then: - ATM_STORE_REL (&w.waiting, 0); - nsync_mu_semaphore_v (&w.sem); */ -typedef struct { - uint32_t tag; /* debug DLL_NSYNC_WAITER, DLL_WAITER, DLL_WAITER_SAMECOND */ - nsync_semaphore sem; /* Thread waits on this semaphore. */ - struct nsync_waiter_s nw; /* An embedded nsync_waiter_s. */ - struct nsync_mu_s_ *cv_mu; /* pointer to nsync_mu associated with a cv wait */ - lock_type - *l_type; /* Lock type of the mu, or nil if not associated with a mu. */ - nsync_atomic_uint32_ remove_count; /* count of removals from queue */ - struct wait_condition_s cond; /* A condition on which to acquire a mu. */ - nsync_dll_element_ same_condition; /* Links neighbours in nw.q with same - non-nil condition. */ - int flags; /* see WAITER_* bits below */ -} waiter; -static const uint32_t WAITER_TAG = 0x0590239f; -static const uint32_t NSYNC_WAITER_TAG = 0x726d2ba9; - -#define WAITER_RESERVED \ - 0x1 /* waiter reserved by a thread, even when not in use */ -#define WAITER_IN_USE 0x2 /* waiter in use by a thread */ - -#define CONTAINER(t_, f_, p_) ((t_ *)(((char *)(p_)) - offsetof(t_, f_))) -#define ASSERT(x) \ - do { \ - if (!(x)) { \ - *(volatile int *)0 = 0; \ - } \ - } while (0) - -/* Return a pointer to the nsync_waiter_s containing nsync_dll_element_ *e. */ -#define DLL_NSYNC_WAITER(e) \ - (NSYNC_DEBUG ? nsync_dll_nsync_waiter_(e) \ - : ((struct nsync_waiter_s *)((e)->container))) -struct nsync_waiter_s *nsync_dll_nsync_waiter_(nsync_dll_element_ *e); - -/* Return a pointer to the waiter struct that *e is embedded in, where *e is an - * nw.q field. */ -#define DLL_WAITER(e) \ - (NSYNC_DEBUG ? nsync_dll_waiter_(e) \ - : CONTAINER(waiter, nw, DLL_NSYNC_WAITER(e))) -waiter *nsync_dll_waiter_(nsync_dll_element_ *e); - -/* Return a pointer to the waiter struct that *e is embedded in, where *e is a - same_condition field. */ -#define DLL_WAITER_SAMECOND(e) \ - (NSYNC_DEBUG ? nsync_dll_waiter_samecond_(e) : ((waiter *)((e)->container))) -waiter *nsync_dll_waiter_samecond_(nsync_dll_element_ *e); - -/* Return a pointer to an unused waiter struct. - Ensures that the enclosed timer is stopped and its channel drained. */ -waiter *nsync_waiter_new_(void); - -/* Return an unused waiter struct *w to the free pool. */ -void nsync_waiter_free_(waiter *w); - -/* ---------- */ - -/* The internals of an nync_note. See internal/note.c for details of locking - discipline. */ -struct nsync_note_s_ { - nsync_dll_element_ - parent_child_link; /* parent's children, under parent->note_mu */ - int expiry_time_valid; /* whether expiry_time is valid; r/o after init */ - nsync_time - expiry_time; /* expiry time, if expiry_time_valid != 0; r/o after init */ - nsync_mu note_mu; /* protects fields below except "notified" */ - nsync_cv no_children_cv; /* signalled when children becomes empty */ - uint32_t disconnecting; /* non-zero => node is being disconnected */ - nsync_atomic_uint32_ notified; /* non-zero if the note has been notified */ - struct nsync_note_s_ *parent; /* points to parent, if any */ - nsync_dll_element_ *children; /* list of children */ - nsync_dll_element_ *waiters; /* list of waiters */ -}; - -/* ---------- */ - -void nsync_mu_lock_slow_(nsync_mu *mu, waiter *w, uint32_t clear, - lock_type *l_type); -void nsync_mu_unlock_slow_(nsync_mu *mu, lock_type *l_type); -nsync_dll_list_ nsync_remove_from_mu_queue_(nsync_dll_list_ mu_queue, - nsync_dll_element_ *e); -void nsync_maybe_merge_conditions_(nsync_dll_element_ *p, - nsync_dll_element_ *n); -nsync_time nsync_note_notified_deadline_(nsync_note n); -int nsync_sem_wait_with_cancel_(waiter *w, nsync_time abs_deadline, - nsync_note cancel_note); - -COSMOPOLITAN_C_END_ -#endif /* !(__ASSEMBLER__ + __LINKER__ + 0) */ -#endif /* NSYNC_COMMON_H_ */ diff --git a/third_party/nsync/counter.h b/third_party/nsync/counter.h deleted file mode 100644 index 7828553e9..000000000 --- a/third_party/nsync/counter.h +++ /dev/null @@ -1,43 +0,0 @@ -#ifndef NSYNC_COUNTER_H_ -#define NSYNC_COUNTER_H_ -#include "third_party/nsync/atomic.h" -#include "third_party/nsync/time.h" -#if !(__ASSEMBLER__ + __LINKER__ + 0) -COSMOPOLITAN_C_START_ - -struct nsync_dll_element_s_; - -/* An nsync_counter represents an unsigned integer that can count up and down, - and wake waiters when zero. */ -typedef struct nsync_counter_s_ *nsync_counter; - -/* Return a freshly allocated nsync_counter with the specified value, - of NULL if an nsync_counter cannot be created. - - Any non-NULL returned value should be passed to nsync_counter_free() when no - longer needed. */ -nsync_counter nsync_counter_new(uint32_t value); - -/* Free resources associated with c. Requires that c was allocated by - nsync_counter_new(), and no concurrent or future operations are applied to - c. */ -void nsync_counter_free(nsync_counter c); - -/* Add delta to c, and return its new value. It is a checkable runtime error - to decrement c below 0, or to increment c (i.e., apply a delta > 0) after a - waiter has waited. */ -uint32_t nsync_counter_add(nsync_counter c, int32_t delta); - -/* Return the current value of c. */ -uint32_t nsync_counter_value(nsync_counter c); - -/* Wait until c has value 0, or until abs_deadline, then return - the value of c. It is a checkable runtime error to increment c after - a waiter may have been woken due to the counter reaching zero. - If abs_deadline==nsync_time_no_deadline, the deadline - is far in the future. */ -uint32_t nsync_counter_wait(nsync_counter c, nsync_time abs_deadline); - -COSMOPOLITAN_C_END_ -#endif /* !(__ASSEMBLER__ + __LINKER__ + 0) */ -#endif /* NSYNC_COUNTER_H_ */ diff --git a/third_party/nsync/cv.h b/third_party/nsync/cv.h deleted file mode 100644 index bd6b5ad91..000000000 --- a/third_party/nsync/cv.h +++ /dev/null @@ -1,157 +0,0 @@ -#ifndef NSYNC_CV_H_ -#define NSYNC_CV_H_ -#include "third_party/nsync/mu.h" -#include "third_party/nsync/time.h" -#if !(__ASSEMBLER__ + __LINKER__ + 0) -COSMOPOLITAN_C_START_ - -#define NSYNC_CV_INIT \ - { NSYNC_ATOMIC_UINT32_INIT_, 0 } - -struct nsync_dll_element_s_; -struct nsync_note_s_; - -/* An nsync_cv is a condition variable in the style of Mesa, Java, - POSIX, and Go's sync.Cond. It allows a thread to wait for a condition - on state protected by a mutex, and to proceed with the mutex held and - the condition true. - - See also nsync_mu_wait() and nsync_mu_wait_with_deadline(), which - implement conditional critical sections. In many cases, they are - easier to use than condition variables. - - Usage - - After making the desired predicate true, call: - - nsync_cv_signal (&cv); // If at most one thread can make use - // of the predicate becoming true. - - or - - nsync_cv_broadcast (&cv); // If multiple threads can make use - // of the predicate becoming true. - - To wait for a predicate with no deadline (assuming - nsync_cv_broadcast() or nsync_cv_signal() is called whenever the - predicate becomes true): - - nsync_mu_lock (μ) - while (!some_predicate_protected_by_mu) { // while-loop required - nsync_cv_wait (&cv, &mu); - } - // predicate is now true - nsync_mu_unlock (&mu); - - To wait for a predicate with a deadline (assuming nsync_cv_broadcast() or - nsync_cv_signal() is called whenever the predicate becomes true): - - nsync_mu_lock (&mu); - while (!some_predicate_protected_by_mu && - nsync_cv_wait_with_deadline (&cv, &mu, abs_deadline, - cancel_note) == 0) { - } - if (some_predicate_protected_by_mu) { // predicate is true - } else { - // predicate is false, and deadline expired, or - // cancel_note was notified. - } - nsync_mu_unlock (&mu); - - or, if the predicate is complex and you wish to write it just once - and inline, you could use the following instead of the for-loop - above: - - nsync_mu_lock (&mu); - int pred_is_true = 0; - int outcome = 0; - while (!(pred_is_true = some_predicate_protected_by_mu) && - outcome == 0) { - outcome = nsync_cv_wait_with_deadline (&cv, &mu, abs_deadline, - cancel_note); - } - if (pred_is_true) { // predicate is true - } else { - // predicate is false, and deadline expired, or - // cancel_note was notified. - } - nsync_mu_unlock (&mu); - - As the examples show, Mesa-style condition variables require that - waits use a loop that tests the predicate anew after each wait. It - may be surprising that these are preferred over the precise wakeups - offered by the condition variables in Hoare monitors. Imprecise - wakeups make more efficient use of the critical section, because - threads can enter it while a woken thread is still emerging from the - scheduler, which may take thousands of cycles. Further, they make the - programme easier to read and debug by making the predicate explicit - locally at the wait, where the predicate is about to be assumed; the - reader does not have to infer the predicate by examining all the - places where wakeups may occur. */ -typedef struct nsync_cv_s_ { - /* see bits below */ - nsync_atomic_uint32_ word; - /* points to tail of list of waiters; under mu. */ - struct nsync_dll_element_s_ *waiters; -} nsync_cv; - -/* An nsync_cv should be zeroed to initialize, which can be accomplished - by initializing with static initializer NSYNC_CV_INIT, or by setting - the entire struct to 0, or using nsync_cv_init(). */ -void nsync_cv_init(nsync_cv *cv); - -/* Wake at least one thread if any are currently blocked on *cv. If the - chosen thread is a reader on an nsync_mu, wake all readers and, if - possible, a writer. */ -void nsync_cv_signal(nsync_cv *cv); - -/* Wake all threads currently blocked on *cv. */ -void nsync_cv_broadcast(nsync_cv *cv); - -/* Atomically release "mu" (which must be held on entry) and block the - caller on *cv. Wait until awakened by a call to nsync_cv_signal() or - nsync_cv_broadcast(), or a spurious wakeup; then reacquire "mu", and - return. Equivalent to a call to nsync_mu_wait_with_deadline() with - abs_deadline==nsync_time_no_deadline, and cancel_note==NULL. Callers - should use nsync_cv_wait() in a loop, as with all standard Mesa-style - condition variables. See examples above. */ -void nsync_cv_wait(nsync_cv *cv, nsync_mu *mu); - -/* Atomically release "mu" (which must be held on entry) and block the - calling thread on *cv. It then waits until awakened by a call to - nsync_cv_signal() or nsync_cv_broadcast() (or a spurious wakeup), or - by the time reaching abs_deadline, or by cancel_note being notified. - In all cases, it reacquires "mu", and returns the reason for the call - returned (0, ETIMEDOUT, or ECANCELED). Use - abs_deadline==nsync_time_no_deadline for no deadline, and - cancel_note==NULL for no cancellation. wait_with_deadline() should be - used in a loop, as with all Mesa-style condition variables. See - examples above. - - There are two reasons for using an absolute deadline, rather than a - relative timeout---these are why pthread_cond_timedwait() also uses - an absolute deadline. First, condition variable waits have to be used - in a loop; with an absolute times, the deadline does not have to be - recomputed on each iteration. Second, in most real programmes, some - activity (such as an RPC to a server, or when guaranteeing response - time in a UI), there is a deadline imposed by the specification or - the caller/user; relative delays can shift arbitrarily with - scheduling delays, and so after multiple waits might extend beyond - the expected deadline. Relative delays tend to be more convenient - mostly in tests and trivial examples than they are in real - programmes. */ -int nsync_cv_wait_with_deadline(nsync_cv *cv, nsync_mu *mu, - nsync_time abs_deadline, - struct nsync_note_s_ *cancel_note); - -/* Like nsync_cv_wait_with_deadline(), but allow an arbitrary lock *v to be - used, given its (*lock)(mu) and (*unlock)(mu) routines. */ -int nsync_cv_wait_with_deadline_generic(nsync_cv *cv, void *mu, - void (*lock)(void *), - void (*unlock)(void *), - nsync_time abs_deadline, - struct nsync_note_s_ *cancel_note); - -COSMOPOLITAN_C_END_ -#endif /* !(__ASSEMBLER__ + __LINKER__ + 0) */ -#endif /* NSYNC_CV_H_ */ diff --git a/third_party/nsync/debug.h b/third_party/nsync/debug.h deleted file mode 100644 index 44d59618f..000000000 --- a/third_party/nsync/debug.h +++ /dev/null @@ -1,39 +0,0 @@ -#ifndef NSYNC_DEBUG_H_ -#define NSYNC_DEBUG_H_ -#include "third_party/nsync/cv.h" -#include "third_party/nsync/mu.h" -#if !(__ASSEMBLER__ + __LINKER__ + 0) -COSMOPOLITAN_C_START_ - -/* Debugging operations for mutexes and condition variables. - - These operations should not be relied upon for normal functionality. - The implementation may be slow, output formats may change, and the - implementation is free to yield the empty string. */ - -/* Place in buf[0,..,n-1] a nul-terminated, human readable string - indicative of some of the internal state of the mutex or condition - variable, and return buf. If n>=4, buffer overflow is indicated by - placing the characters "..." at the end of the string. - - The *_and_waiters() variants attempt to output the waiter lists in - addition to the basic state. These variants may acquire internal - locks and follow internal pointers. Thus, they are riskier if invoked - in an address space whose overall health is uncertain. */ -char *nsync_mu_debug_state(nsync_mu *mu, char *buf, int n); -char *nsync_cv_debug_state(nsync_cv *cv, char *buf, int n); -char *nsync_mu_debug_state_and_waiters(nsync_mu *mu, char *buf, int n); -char *nsync_cv_debug_state_and_waiters(nsync_cv *cv, char *buf, int n); - -/* Like nsync_*_debug_state_and_waiters(), but ignoring all locking and - safety considerations, and using an internal, possibly static buffer - that may be overwritten by subsequent or concurrent calls to these - routines. These variants should be used only from an interactive - debugger, when all other threads are stopped; the debugger is - expected to recover from errors. */ -char *nsync_mu_debugger(nsync_mu *mu); -char *nsync_cv_debugger(nsync_cv *cv); - -COSMOPOLITAN_C_END_ -#endif /* !(__ASSEMBLER__ + __LINKER__ + 0) */ -#endif /* NSYNC_DEBUG_H_ */ diff --git a/third_party/nsync/dll.c b/third_party/nsync/dll.c deleted file mode 100644 index d5ff5e555..000000000 --- a/third_party/nsync/dll.c +++ /dev/null @@ -1,143 +0,0 @@ -/*-*- mode:c;indent-tabs-mode:t;c-basic-offset:8;tab-width:8;coding:utf-8 -*-│ -│vi: set et ft=c ts=8 tw=8 fenc=utf-8 :vi│ -╞══════════════════════════════════════════════════════════════════════════════╡ -│ Copyright 2016 Google Inc. │ -│ │ -│ Licensed under the Apache License, Version 2.0 (the "License"); │ -│ you may not use this file except in compliance with the License. │ -│ You may obtain a copy of the License at │ -│ │ -│ http://www.apache.org/licenses/LICENSE-2.0 │ -│ │ -│ Unless required by applicable law or agreed to in writing, software │ -│ distributed under the License is distributed on an "AS IS" BASIS, │ -│ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. │ -│ See the License for the specific language governing permissions and │ -│ limitations under the License. │ -╚─────────────────────────────────────────────────────────────────────────────*/ -#include "third_party/nsync/dll.h" - -asm(".ident\t\"\\n\\n\ -*NSYNC (Apache 2.0)\\n\ -Copyright 2016 Google, Inc.\\n\ -https://github.com/google/nsync\""); -// clang-format off - -/* Initialize *e. */ -void nsync_dll_init_ (nsync_dll_element_ *e, void *container) { - e->next = e; - e->prev = e; - e->container = container; -} - -/* Return whether list is empty. */ -int nsync_dll_is_empty_ (nsync_dll_list_ list) { - return (list == NULL); -} - -/* Remove *e from list, and returns the new list. */ -nsync_dll_list_ nsync_dll_remove_ (nsync_dll_list_ list, nsync_dll_element_ *e) { - if (list == e) { /* removing tail of list */ - if (list->prev == list) { - list = NULL; /* removing only element of list */ - } else { - list = list->prev; - } - } - e->next->prev = e->prev; - e->prev->next = e->next; - e->next = e; - e->prev = e; - return (list); -} - -/* Cause element *n and its successors to come after element *p. - Requires n and p are non-NULL and do not point at elements of the same list. - - Unlike the other operations in this API, this operation acts on - two circular lists of elements, rather than on a "head" location that points - to such a circular list. - - If the two lists are p->p_2nd->p_mid->p_last->p and n->n_2nd->n_mid->n_last->n, - then after nsync_dll_splice_after_ (p, n), the p list would be: - p->n->n_2nd->n_mid->n_last->p_2nd->p_mid->p_last->p. */ -void nsync_dll_splice_after_ (nsync_dll_element_ *p, nsync_dll_element_ *n) { - nsync_dll_element_ *p_2nd = p->next; - nsync_dll_element_ *n_last = n->prev; - p->next = n; /* n follows p */ - n->prev = p; - n_last->next = p_2nd; /* remainder of p-list follows last of n-list */ - p_2nd->prev = n_last; -} - -/* Make element *e the first element of list, and return - the list. The resulting list will have *e as its first element, followed by - any elements in the same list as *e, followed by the elements that were - previously in list. Requires that *e not be in list. If e==NULL, list is - returned unchanged. - - Suppose the e list is e->e_2nd->e_mid->e_last->e. - Recall that a head "list" points to the last element of its list. - If list is initially null, then the outcome is: - result = e_last->e->e_2nd->e_mid->e_last - If list is initially list->list_last->list_1st->list_mid->list_last, - then the outcome is: - result = list_last->e->e_2nd->e_mid->e_last->list_1st->list_mid->list_last - */ -nsync_dll_list_ nsync_dll_make_first_in_list_ (nsync_dll_list_ list, nsync_dll_element_ *e) { - if (e != NULL) { - if (list == NULL) { - list = e->prev; /*e->prev is e_last*/ - } else { - nsync_dll_splice_after_ (list, e); - } - } - return (list); -} - -/* Make element *e the last element of list, and return - the list. The resulting list will have *e as its last element, preceded by - any elements in the same list as *e, preceded by the elements that were - previously in list. Requires that *e not be in list. If e==NULL, list is - returned unchanged. */ -nsync_dll_list_ nsync_dll_make_last_in_list_ (nsync_dll_list_ list, nsync_dll_element_ *e) { - if (e != NULL) { - nsync_dll_make_first_in_list_ (list, e->next); - list = e; - } - return (list); -} - -/* Return a pointer to the first element of list, or NULL if list is empty. */ -nsync_dll_element_ *nsync_dll_first_ (nsync_dll_list_ list) { - nsync_dll_element_ *first = NULL; - if (list != NULL) { - first = list->next; - } - return (first); -} - -/* Return a pointer to the last element of list, or NULL if list is empty. */ -nsync_dll_element_ *nsync_dll_last_ (nsync_dll_list_ list) { - return (list); -} - -/* Return a pointer to the next element of list following *e, - or NULL if there is no such element. */ -nsync_dll_element_ *nsync_dll_next_ (nsync_dll_list_ list, nsync_dll_element_ *e) { - nsync_dll_element_ *next = NULL; - if (e != list) { - next = e->next; - } - return (next); -} - -/* Return a pointer to the previous element of list following *e, - or NULL if there is no such element. */ -nsync_dll_element_ *nsync_dll_prev_ (nsync_dll_list_ list, nsync_dll_element_ *e) { - nsync_dll_element_ *prev = NULL; - if (e != list->next) { - prev = e->prev; - } - return (prev); -} diff --git a/third_party/nsync/dll.h b/third_party/nsync/dll.h deleted file mode 100644 index a1345d386..000000000 --- a/third_party/nsync/dll.h +++ /dev/null @@ -1,69 +0,0 @@ -#ifndef NSYNC_DLL_H_ -#define NSYNC_DLL_H_ -#if !(__ASSEMBLER__ + __LINKER__ + 0) -COSMOPOLITAN_C_START_ - -/* A nsync_dll_element_ represents an element of a doubly-linked list of - waiters. */ -typedef struct nsync_dll_element_s_ { - struct nsync_dll_element_s_ *next; - struct nsync_dll_element_s_ *prev; - /* points to the struct this nsync_dll struct is embedded in. */ - void *container; -} nsync_dll_element_; - -/* A nsync_dll_list_ represents a list of nsync_dll_elements_. */ -typedef nsync_dll_element_ *nsync_dll_list_; /* last elem of circular list; nil - => empty; first is x.next. */ - -/* Initialize *e. */ -void nsync_dll_init_(nsync_dll_element_ *e, void *container); - -/* Return whether list is empty. */ -int nsync_dll_is_empty_(nsync_dll_list_ list); - -/* Remove *e from list, and returns the new list. */ -nsync_dll_list_ nsync_dll_remove_(nsync_dll_list_ list, nsync_dll_element_ *e); - -/* Cause element *n and its successors to come after element *p. - Requires n and p are non-NULL and do not point at elements of the - same list. */ -void nsync_dll_splice_after_(nsync_dll_element_ *p, nsync_dll_element_ *n); - -/* Make element *e the first element of list, and return the list. The - resulting list will have *e as its first element, followed by any - elements in the same list as *e, followed by the elements that were - previously in list. Requires that *e not be in list. If e==NULL, list - is returned unchanged. */ -nsync_dll_list_ nsync_dll_make_first_in_list_(nsync_dll_list_ list, - nsync_dll_element_ *e); - -/* Make element *e the last element of list, and return the list. The - resulting list will have *e as its last element, preceded by any - elements in the same list as *e, preceded by the elements that were - previously in list. Requires that *e not be in list. If e==NULL, list - is returned unchanged. */ -nsync_dll_list_ nsync_dll_make_last_in_list_(nsync_dll_list_ list, - nsync_dll_element_ *e); - -/* Return a pointer to the first element of list, or NULL if list is - * empty. */ -nsync_dll_element_ *nsync_dll_first_(nsync_dll_list_ list); - -/* Return a pointer to the last element of list, or NULL if list is - * empty. */ -nsync_dll_element_ *nsync_dll_last_(nsync_dll_list_ list); - -/* Return a pointer to the next element of list following *e, or NULL if - there is no such element. */ -nsync_dll_element_ *nsync_dll_next_(nsync_dll_list_ list, - nsync_dll_element_ *e); - -/* Return a pointer to the previous element of list following *e, or - NULL if there is no such element. */ -nsync_dll_element_ *nsync_dll_prev_(nsync_dll_list_ list, - nsync_dll_element_ *e); - -COSMOPOLITAN_C_END_ -#endif /* !(__ASSEMBLER__ + __LINKER__ + 0) */ -#endif /* NSYNC_DLL_H_ */ diff --git a/third_party/nsync/futex.c b/third_party/nsync/futex.c deleted file mode 100644 index cc875abbe..000000000 --- a/third_party/nsync/futex.c +++ /dev/null @@ -1,124 +0,0 @@ -/*-*- mode:c;indent-tabs-mode:t;c-basic-offset:8;tab-width:8;coding:utf-8 -*-│ -│vi: set et ft=c ts=8 tw=8 fenc=utf-8 :vi│ -╞══════════════════════════════════════════════════════════════════════════════╡ -│ Copyright 2022 Justine Alexandra Roberts Tunney │ -│ │ -│ Permission to use, copy, modify, and/or distribute this software for │ -│ any purpose with or without fee is hereby granted, provided that the │ -│ above copyright notice and this permission notice appear in all copies. │ -│ │ -│ THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL │ -│ WARRANTIES WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED │ -│ WARRANTIES OF MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE │ -│ AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL │ -│ DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR │ -│ PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER │ -│ TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR │ -│ PERFORMANCE OF THIS SOFTWARE. │ -╚─────────────────────────────────────────────────────────────────────────────*/ -#include "libc/calls/strace.internal.h" -#include "libc/calls/struct/timespec.internal.h" -#include "libc/dce.h" -#include "libc/errno.h" -#include "libc/intrin/describeflags.internal.h" -#include "libc/sysv/consts/futex.h" -#include "libc/thread/thread.h" -#include "third_party/nsync/common.internal.h" -#include "third_party/nsync/futex.internal.h" -// clang-format off - -/* futex() polyfill w/ sched_yield() fallback */ - -#define FUTEX_WAIT_BITS_ FUTEX_BITSET_MATCH_ANY - -int _futex (int *, int, int, const struct timespec *, int *, int); - -static int FUTEX_WAIT_; -static int FUTEX_WAKE_; -static int FUTEX_PRIVATE_FLAG_; -static bool FUTEX_IS_SUPPORTED; -bool FUTEX_TIMEOUT_IS_ABSOLUTE; - -__attribute__((__constructor__)) static void sync_futex_init_ (void) { - int x = 0; - - FUTEX_WAKE_ = FUTEX_WAKE; - - if (IsLinux () && - _futex (&x, FUTEX_WAIT_BITSET, 1, 0, 0, - FUTEX_BITSET_MATCH_ANY) == -EAGAIN) { - FUTEX_WAIT_ = FUTEX_WAIT_BITSET; - FUTEX_TIMEOUT_IS_ABSOLUTE = true; - } else { - FUTEX_WAIT_ = FUTEX_WAIT; - } - - if (IsOpenbsd () || - (IsLinux () && - !_futex (&x, FUTEX_WAKE_PRIVATE, 1, 0, 0, 0))) { - FUTEX_PRIVATE_FLAG_ = FUTEX_PRIVATE_FLAG; - } - - // In our testing, we found that the monotonic clock on various - // popular systems (such as Linux, and some BSD variants) was no - // better behaved than the realtime clock, and routinely took - // large steps backwards, especially on multiprocessors. Given - // that "monotonic" doesn't seem to mean what it says, - // implementers of nsync_time might consider retaining the - // simplicity of a single epoch within an address space, by - // configuring any time synchronization mechanism (like ntp) to - // adjust for leap seconds by adjusting the rate, rather than - // with a backwards step. - if (IsLinux () && - _futex (&x, FUTEX_WAIT_BITSET | FUTEX_CLOCK_REALTIME, - 1, 0, 0, FUTEX_BITSET_MATCH_ANY) == -EAGAIN) { - FUTEX_WAIT_ |= FUTEX_CLOCK_REALTIME; - } - - FUTEX_IS_SUPPORTED = IsLinux() || IsOpenbsd(); -} - -int nsync_futex_wait_ (int *p, int expect, char pshare, struct timespec *timeout) { - int rc, op; - if (FUTEX_IS_SUPPORTED) { - op = FUTEX_WAIT_; - if (pshare == PTHREAD_PROCESS_PRIVATE) { - op |= FUTEX_PRIVATE_FLAG_; - } - rc = _futex (p, op, expect, timeout, 0, FUTEX_WAIT_BITS_); - if (IsOpenbsd() && rc > 0) { - // [jart] openbsd does this without setting carry flag - rc = -rc; - } - STRACE("futex(%t, %s, %d, %s) → %s", - p, DescribeFutexOp(op), expect, - DescribeTimespec(0, timeout), DescribeFutexResult(rc)); - } else { - nsync_yield_ (); - if (timeout) { - rc = -ETIMEDOUT; - } else { - rc = 0; - } - } - return rc; -} - -int nsync_futex_wake_ (int *p, int count, char pshare) { - int rc, op; - int wake (void *, int, int) asm ("_futex"); - if (FUTEX_IS_SUPPORTED) { - op = FUTEX_WAKE_; - if (pshare == PTHREAD_PROCESS_PRIVATE) { - op |= FUTEX_PRIVATE_FLAG_; - } - rc = wake (p, op, count); - STRACE("futex(%t, %s, %d) → %s", p, - DescribeFutexOp(op), - count, DescribeFutexResult(rc)); - } else { - nsync_yield_ (); - rc = 0; - } - return rc; -} diff --git a/third_party/nsync/futex.internal.h b/third_party/nsync/futex.internal.h deleted file mode 100644 index 5fc8f4998..000000000 --- a/third_party/nsync/futex.internal.h +++ /dev/null @@ -1,15 +0,0 @@ -#ifndef NSYNC_FUTEX_INTERNAL_H_ -#define NSYNC_FUTEX_INTERNAL_H_ -#include "libc/calls/struct/timespec.h" -#include "libc/dce.h" -#if !(__ASSEMBLER__ + __LINKER__ + 0) -COSMOPOLITAN_C_START_ - -extern bool FUTEX_TIMEOUT_IS_ABSOLUTE; - -int nsync_futex_wake_(int *, int, char); -int nsync_futex_wait_(int *, int, char, struct timespec *); - -COSMOPOLITAN_C_END_ -#endif /* !(__ASSEMBLER__ + __LINKER__ + 0) */ -#endif /* NSYNC_FUTEX_INTERNAL_H_ */ diff --git a/third_party/nsync/malloc.c b/third_party/nsync/malloc.c deleted file mode 100644 index 65708ddfe..000000000 --- a/third_party/nsync/malloc.c +++ /dev/null @@ -1,49 +0,0 @@ -/*-*- mode:c;indent-tabs-mode:t;c-basic-offset:8;tab-width:8;coding:utf-8 -*-│ -│vi: set et ft=c ts=8 tw=8 fenc=utf-8 :vi│ -╞══════════════════════════════════════════════════════════════════════════════╡ -│ Copyright 2022 Justine Alexandra Roberts Tunney │ -│ │ -│ Permission to use, copy, modify, and/or distribute this software for │ -│ any purpose with or without fee is hereby granted, provided that the │ -│ above copyright notice and this permission notice appear in all copies. │ -│ │ -│ THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL │ -│ WARRANTIES WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED │ -│ WARRANTIES OF MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE │ -│ AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL │ -│ DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR │ -│ PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER │ -│ TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR │ -│ PERFORMANCE OF THIS SOFTWARE. │ -╚─────────────────────────────────────────────────────────────────────────────*/ -#include "libc/atomic.h" -#include "libc/calls/extend.internal.h" -#include "libc/intrin/atomic.h" -#include "libc/macros.internal.h" -#include "third_party/nsync/common.internal.h" -#include "third_party/nsync/malloc.internal.h" -// clang-format off - -static char *nsync_malloc_endptr_; -static size_t nsync_malloc_total_; -static atomic_char nsync_malloc_lock_; - -/* nsync_malloc_() is a malloc-like routine used by mutex and condition - variable code to allocate waiter structs. This allows *NSYNC mutexes - to be used by malloc(), by providing another, simpler allocator here. - The intent is that the implicit NULL value here can be overridden by - a client declaration that uses an initializer. */ -void *nsync_malloc_ (size_t size) { - char *start; - size_t offset; - size = ROUNDUP (size, __BIGGEST_ALIGNMENT__); - while (atomic_exchange (&nsync_malloc_lock_, 1)) nsync_yield_ (); - offset = nsync_malloc_total_; - nsync_malloc_total_ += size; - start = (char *) 0x6fc000040000; - if (!nsync_malloc_endptr_) nsync_malloc_endptr_ = start; - nsync_malloc_endptr_ = _extend (start, nsync_malloc_total_, - nsync_malloc_endptr_, 0x6fcfffff0000); - atomic_store_explicit (&nsync_malloc_lock_, 0, memory_order_relaxed); - return start + offset; -} diff --git a/third_party/nsync/malloc.internal.h b/third_party/nsync/malloc.internal.h deleted file mode 100644 index e4453e410..000000000 --- a/third_party/nsync/malloc.internal.h +++ /dev/null @@ -1,10 +0,0 @@ -#ifndef NSYNC_MALLOC_INTERNAL_H_ -#define NSYNC_MALLOC_INTERNAL_H_ -#if !(__ASSEMBLER__ + __LINKER__ + 0) -COSMOPOLITAN_C_START_ - -void *nsync_malloc_(size_t); - -COSMOPOLITAN_C_END_ -#endif /* !(__ASSEMBLER__ + __LINKER__ + 0) */ -#endif /* NSYNC_MALLOC_INTERNAL_H_ */ diff --git a/third_party/nsync/mu.c b/third_party/nsync/mu.c deleted file mode 100644 index bda08494d..000000000 --- a/third_party/nsync/mu.c +++ /dev/null @@ -1,548 +0,0 @@ -/*-*- mode:c;indent-tabs-mode:t;c-basic-offset:8;tab-width:8;coding:utf-8 -*-│ -│vi: set et ft=c ts=8 tw=8 fenc=utf-8 :vi│ -╞══════════════════════════════════════════════════════════════════════════════╡ -│ Copyright 2016 Google Inc. │ -│ │ -│ Licensed under the Apache License, Version 2.0 (the "License"); │ -│ you may not use this file except in compliance with the License. │ -│ You may obtain a copy of the License at │ -│ │ -│ http://www.apache.org/licenses/LICENSE-2.0 │ -│ │ -│ Unless required by applicable law or agreed to in writing, software │ -│ distributed under the License is distributed on an "AS IS" BASIS, │ -│ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. │ -│ See the License for the specific language governing permissions and │ -│ limitations under the License. │ -╚─────────────────────────────────────────────────────────────────────────────*/ -#include "libc/str/str.h" -#include "third_party/nsync/atomic.h" -#include "third_party/nsync/common.internal.h" -#include "third_party/nsync/dll.h" -#include "third_party/nsync/mu_semaphore.h" -#include "third_party/nsync/races.internal.h" -#include "third_party/nsync/wait_s.internal.h" - -asm(".ident\t\"\\n\\n\ -*NSYNC (Apache 2.0)\\n\ -Copyright 2016 Google, Inc.\\n\ -https://github.com/google/nsync\""); -// clang-format off - -/* Initialize *mu. */ -void nsync_mu_init (nsync_mu *mu) { - memset ((void *) mu, 0, sizeof (*mu)); -} - -/* Release the mutex spinlock. */ -static void mu_release_spinlock (nsync_mu *mu) { - uint32_t old_word = ATM_LOAD (&mu->word); - while (!ATM_CAS_REL (&mu->word, old_word, old_word & ~MU_SPINLOCK)) { - old_word = ATM_LOAD (&mu->word); - } -} - -/* Lock *mu using the specified lock_type, waiting on *w if necessary. - "clear" should be zero if the thread has not previously slept on *mu, and - MU_DESIG_WAKER if it has; this represents bits that nsync_mu_lock_slow_() must clear when - it either acquires or sleeps on *mu. The caller owns *w on return; it is in a valid - state to be returned to the free pool. */ -void nsync_mu_lock_slow_ (nsync_mu *mu, waiter *w, uint32_t clear, lock_type *l_type) { - uint32_t zero_to_acquire; - uint32_t wait_count; - uint32_t long_wait; - unsigned attempts = 0; /* attempt count; used for spinloop backoff */ - w->cv_mu = NULL; /* not a cv wait */ - w->cond.f = NULL; /* Not using a conditional critical section. */ - w->cond.v = NULL; - w->cond.eq = NULL; - w->l_type = l_type; - zero_to_acquire = l_type->zero_to_acquire; - if (clear != 0) { - /* Only the constraints of mutual exclusion should stop a designated waker. */ - zero_to_acquire &= ~(MU_WRITER_WAITING | MU_LONG_WAIT); - } - wait_count = 0; /* number of times we waited, and were woken. */ - long_wait = 0; /* set to MU_LONG_WAIT when wait_count gets large */ - for (;;) { - uint32_t old_word = ATM_LOAD (&mu->word); - if ((old_word & zero_to_acquire) == 0) { - /* lock can be acquired; try to acquire, possibly - clearing MU_DESIG_WAKER and MU_LONG_WAIT. */ - if (ATM_CAS_ACQ (&mu->word, old_word, - (old_word+l_type->add_to_acquire) & - ~(clear|long_wait|l_type->clear_on_acquire))) { - return; - } - } else if ((old_word&MU_SPINLOCK) == 0 && - ATM_CAS_ACQ (&mu->word, old_word, - (old_word|MU_SPINLOCK|long_wait| - l_type->set_when_waiting) & ~(clear | MU_ALL_FALSE))) { - - /* Spinlock is now held, and lock is held by someone - else; MU_WAITING has also been set; queue ourselves. - There's no need to adjust same_condition here, - because w.condition==NULL. */ - ATM_STORE (&w->nw.waiting, 1); - if (wait_count == 0) { - /* first wait goes to end of queue */ - mu->waiters = nsync_dll_make_last_in_list_ (mu->waiters, - &w->nw.q); - } else { - /* subsequent waits go to front of queue */ - mu->waiters = nsync_dll_make_first_in_list_ (mu->waiters, - &w->nw.q); - } - - /* Release spinlock. Cannot use a store here, because - the current thread does not hold the mutex. If - another thread were a designated waker, the mutex - holder could be concurrently unlocking, even though - we hold the spinlock. */ - mu_release_spinlock (mu); - - /* wait until awoken. */ - while (ATM_LOAD_ACQ (&w->nw.waiting) != 0) { /* acquire load */ - nsync_mu_semaphore_p (&w->sem); - } - wait_count++; - /* If the thread has been woken more than this many - times, and still not acquired, it sets the - MU_LONG_WAIT bit to prevent thread that have not - waited from acquiring. This is the starvation - avoidance mechanism. The number is fairly high so - that we continue to benefit from the throughput of - not having running threads wait unless absolutely - necessary. */ - if (wait_count == LONG_WAIT_THRESHOLD) { /* repeatedly woken */ - long_wait = MU_LONG_WAIT; /* force others to wait at least once */ - } - - attempts = 0; - clear = MU_DESIG_WAKER; - /* Threads that have been woken at least once don't care - about waiting writers or long waiters. */ - zero_to_acquire &= ~(MU_WRITER_WAITING | MU_LONG_WAIT); - } - attempts = nsync_spin_delay_ (attempts); - } -} - -/* Attempt to acquire *mu in writer mode without blocking, and return non-zero - iff successful. Return non-zero with high probability if *mu was free on - entry. */ -int nsync_mu_trylock (nsync_mu *mu) { - int result; - IGNORE_RACES_START (); - if (ATM_CAS_ACQ (&mu->word, 0, MU_WADD_TO_ACQUIRE)) { /* acquire CAS */ - result = 1; - } else { - uint32_t old_word = ATM_LOAD (&mu->word); - result = ((old_word & MU_WZERO_TO_ACQUIRE) == 0 && - ATM_CAS_ACQ (&mu->word, old_word, - (old_word + MU_WADD_TO_ACQUIRE) & ~MU_WCLEAR_ON_ACQUIRE)); - } - IGNORE_RACES_END (); - return (result); -} - -/* Block until *mu is free and then acquire it in writer mode. */ -void nsync_mu_lock (nsync_mu *mu) { - IGNORE_RACES_START (); - if (!ATM_CAS_ACQ (&mu->word, 0, MU_WADD_TO_ACQUIRE)) { /* acquire CAS */ - uint32_t old_word = ATM_LOAD (&mu->word); - if ((old_word&MU_WZERO_TO_ACQUIRE) != 0 || - !ATM_CAS_ACQ (&mu->word, old_word, - (old_word+MU_WADD_TO_ACQUIRE) & ~MU_WCLEAR_ON_ACQUIRE)) { - waiter *w = nsync_waiter_new_ (); - nsync_mu_lock_slow_ (mu, w, 0, nsync_writer_type_); - nsync_waiter_free_ (w); - } - } - IGNORE_RACES_END (); -} - -/* Attempt to acquire *mu in reader mode without blocking, and return non-zero - iff successful. Returns non-zero with high probability if *mu was free on - entry. It may fail to acquire if a writer is waiting, to avoid starvation. - */ -int nsync_mu_rtrylock (nsync_mu *mu) { - int result; - IGNORE_RACES_START (); - if (ATM_CAS_ACQ (&mu->word, 0, MU_RADD_TO_ACQUIRE)) { /* acquire CAS */ - result = 1; - } else { - uint32_t old_word = ATM_LOAD (&mu->word); - result = ((old_word&MU_RZERO_TO_ACQUIRE) == 0 && - ATM_CAS_ACQ (&mu->word, old_word, - (old_word+MU_RADD_TO_ACQUIRE) & ~MU_RCLEAR_ON_ACQUIRE)); - } - IGNORE_RACES_END (); - return (result); -} - -/* Block until *mu can be acquired in reader mode and then acquire it. */ -void nsync_mu_rlock (nsync_mu *mu) { - IGNORE_RACES_START (); - if (!ATM_CAS_ACQ (&mu->word, 0, MU_RADD_TO_ACQUIRE)) { /* acquire CAS */ - uint32_t old_word = ATM_LOAD (&mu->word); - if ((old_word&MU_RZERO_TO_ACQUIRE) != 0 || - !ATM_CAS_ACQ (&mu->word, old_word, - (old_word+MU_RADD_TO_ACQUIRE) & ~MU_RCLEAR_ON_ACQUIRE)) { - waiter *w = nsync_waiter_new_ (); - nsync_mu_lock_slow_ (mu, w, 0, nsync_reader_type_); - nsync_waiter_free_ (w); - } - } - IGNORE_RACES_END (); -} - -/* Invoke the condition associated with *p, which is an element of - a "waiter" list. */ -static int condition_true (nsync_dll_element_ *p) { - return ((*DLL_WAITER (p)->cond.f) (DLL_WAITER (p)->cond.v)); -} - -/* If *p is an element of waiter_list (a list of "waiter" structs(, return a - pointer to the next element of the list that has a different condition. */ -static nsync_dll_element_ *skip_past_same_condition ( - nsync_dll_list_ waiter_list, nsync_dll_element_ *p) { - nsync_dll_element_ *next; - nsync_dll_element_ *last_with_same_condition = - &DLL_WAITER_SAMECOND (DLL_WAITER (p)->same_condition.prev)->nw.q; - if (last_with_same_condition != p && last_with_same_condition != p->prev) { - /* First in set with same condition, so skip to end. */ - next = nsync_dll_next_ (waiter_list, last_with_same_condition); - } else { - next = nsync_dll_next_ (waiter_list, p); - } - return (next); -} - -/* Merge the same_condition lists of *p and *n if they have the same non-NULL - condition. */ -void nsync_maybe_merge_conditions_ (nsync_dll_element_ *p, nsync_dll_element_ *n) { - if (p != NULL && n != NULL && - WAIT_CONDITION_EQ (&DLL_WAITER (p)->cond, &DLL_WAITER (n)->cond)) { - nsync_dll_splice_after_ (&DLL_WAITER (p)->same_condition, - &DLL_WAITER (n)->same_condition); - } -} - -/* Remove element *e from nsync_mu waiter queue mu_queue, fixing - up the same_condition list by merging the lists on either side if possible. - Also increment the waiter's remove_count. */ -nsync_dll_list_ nsync_remove_from_mu_queue_ (nsync_dll_list_ mu_queue, nsync_dll_element_ *e) { - /* Record previous and next elements in the original queue. */ - nsync_dll_element_ *prev = e->prev; - nsync_dll_element_ *next = e->next; - uint32_t old_value; - /* Remove. */ - mu_queue = nsync_dll_remove_ (mu_queue, e); - do { - old_value = ATM_LOAD (&DLL_WAITER (e)->remove_count); - } while (!ATM_CAS (&DLL_WAITER (e)->remove_count, old_value, old_value+1)); - if (!nsync_dll_is_empty_ (mu_queue)) { - /* Fix up same_condition. */ - nsync_dll_element_ *e_same_condition = &DLL_WAITER (e)->same_condition; - - if (e_same_condition->next != e_same_condition) { - /* *e is linked to a same_condition neighbour---just remove it. */ - e_same_condition->next->prev = e_same_condition->prev; - e_same_condition->prev->next = e_same_condition->next; - e_same_condition->next = e_same_condition; - e_same_condition->prev = e_same_condition; - } else if (prev != nsync_dll_last_ (mu_queue)) { - /* Merge the new neighbours together if we can. */ - nsync_maybe_merge_conditions_ (prev, next); - } - } - return (mu_queue); -} - -/* Unlock *mu and wake one or more waiters as appropriate after an unlock. - It is called with *mu held in mode l_type. */ -void nsync_mu_unlock_slow_ (nsync_mu *mu, lock_type *l_type) { - unsigned attempts = 0; /* attempt count; used for backoff */ - for (;;) { - uint32_t old_word = ATM_LOAD (&mu->word); - int testing_conditions = ((old_word & MU_CONDITION) != 0); - uint32_t early_release_mu = l_type->add_to_acquire; - uint32_t late_release_mu = 0; - if (testing_conditions) { - /* Convert to a writer lock, and release later. - - A writer lock is currently needed to test conditions - because exclusive access is needed to the list to - allow modification. The spinlock cannot be used - to achieve that, because an internal lock should not - be held when calling the external predicates. - - We must test conditions even though a reader region - cannot have made any new ones true because some - might have been true before the reader region started. - The MU_ALL_FALSE test below shortcuts the case where - the conditions are known all to be false. */ - early_release_mu = l_type->add_to_acquire - MU_WLOCK; - late_release_mu = MU_WLOCK; - } - if ((old_word&MU_WAITING) == 0 || (old_word&MU_DESIG_WAKER) != 0 || - (old_word & MU_RLOCK_FIELD) > MU_RLOCK || - (old_word & (MU_RLOCK|MU_ALL_FALSE)) == (MU_RLOCK|MU_ALL_FALSE)) { - /* no one to wake, there's a designated waker waking - up, there are still readers, or it's a reader and all waiters - have false conditions */ - if (ATM_CAS_REL (&mu->word, old_word, - (old_word - l_type->add_to_acquire) & - ~l_type->clear_on_uncontended_release)) { - return; - } - } else if ((old_word&MU_SPINLOCK) == 0 && - ATM_CAS_ACQ (&mu->word, old_word, - (old_word-early_release_mu)|MU_SPINLOCK|MU_DESIG_WAKER)) { - nsync_dll_list_ wake; - lock_type *wake_type; - uint32_t clear_on_release; - uint32_t set_on_release; - /* The spinlock is now held, and we've set the - designated wake flag, since we're likely to wake a - thread that will become that designated waker. If - there are conditions to check, the mutex itself is - still held. */ - - nsync_dll_element_ *p = NULL; - nsync_dll_element_ *next = NULL; - - /* Swap the entire mu->waiters list into the local - "new_waiters" list. This gives us exclusive access - to the list, even if we unlock the spinlock, which - we may do if checking conditions. The loop below - will grab more new waiters that arrived while we - were checking conditions, and terminates only if no - new waiters arrive in one loop iteration. */ - nsync_dll_list_ waiters = NULL; - nsync_dll_list_ new_waiters = mu->waiters; - mu->waiters = NULL; - - /* Remove a waiter from the queue, if possible. */ - wake = NULL; /* waiters to wake. */ - wake_type = NULL; /* type of waiter(s) on wake, or NULL if wake is empty. */ - clear_on_release = MU_SPINLOCK; - set_on_release = MU_ALL_FALSE; - while (!nsync_dll_is_empty_ (new_waiters)) { /* some new waiters to consider */ - p = nsync_dll_first_ (new_waiters); - if (testing_conditions) { - /* Should we continue to test conditions? */ - if (wake_type == nsync_writer_type_) { - /* No, because we're already waking a writer, - and need wake no others.*/ - testing_conditions = 0; - } else if (wake_type == NULL && - DLL_WAITER (p)->l_type != nsync_reader_type_ && - DLL_WAITER (p)->cond.f == NULL) { - /* No, because we've woken no one, but the - first waiter is a writer with no condition, - so we will certainly wake it, and need wake - no others. */ - testing_conditions = 0; - } - } - /* If testing waiters' conditions, release the - spinlock while still holding the write lock. - This is so that the spinlock is not held - while the conditions are evaluated. */ - if (testing_conditions) { - mu_release_spinlock (mu); - } - - /* Process the new waiters picked up in this iteration of the - "while (!nsync_dll_is_empty_ (new_waiters))" loop, - and stop looking when we run out of waiters, or we find - a writer to wake up. */ - while (p != NULL && wake_type != nsync_writer_type_) { - int p_has_condition; - next = nsync_dll_next_ (new_waiters, p); - p_has_condition = (DLL_WAITER (p)->cond.f != NULL); - if (p_has_condition && !testing_conditions) { - nsync_panic_ ("checking a waiter condition " - "while unlocked\n"); - } - if (p_has_condition && !condition_true (p)) { - /* condition is false */ - /* skip to the end of the same_condition group. */ - next = skip_past_same_condition (new_waiters, p); - } else if (wake_type == NULL || - DLL_WAITER (p)->l_type == nsync_reader_type_) { - /* Wake this thread. */ - new_waiters = nsync_remove_from_mu_queue_ ( - new_waiters, p); - wake = nsync_dll_make_last_in_list_ (wake, p); - wake_type = DLL_WAITER (p)->l_type; - } else { - /* Failing to wake a writer - that could acquire if it - were first. */ - set_on_release |= MU_WRITER_WAITING; - set_on_release &= ~MU_ALL_FALSE; - } - p = next; - } - - if (p != NULL) { - /* Didn't search to end of list, so can't be sure - all conditions are false. */ - set_on_release &= ~MU_ALL_FALSE; - } - - /* If testing waiters' conditions, reacquire the spinlock - released above. */ - if (testing_conditions) { - nsync_spin_test_and_set_ (&mu->word, MU_SPINLOCK, - MU_SPINLOCK, 0); - } - - /* add the new_waiters to the last of the waiters. */ - nsync_maybe_merge_conditions_ (nsync_dll_last_ (waiters), - nsync_dll_first_ (new_waiters)); - waiters = nsync_dll_make_last_in_list_ (waiters, - nsync_dll_last_ (new_waiters)); - /* Pick up the next set of new waiters. */ - new_waiters = mu->waiters; - mu->waiters = NULL; - } - - /* Return the local waiter list to *mu. */ - mu->waiters = waiters; - - if (nsync_dll_is_empty_ (wake)) { - /* not waking a waiter => no designated waker */ - clear_on_release |= MU_DESIG_WAKER; - } - - if ((set_on_release & MU_ALL_FALSE) == 0) { - /* If not explicitly setting MU_ALL_FALSE, clear it. */ - clear_on_release |= MU_ALL_FALSE; - } - - if (nsync_dll_is_empty_ (mu->waiters)) { - /* no waiters left */ - clear_on_release |= MU_WAITING | MU_WRITER_WAITING | - MU_CONDITION | MU_ALL_FALSE; - } - - /* Release the spinlock, and possibly the lock if - late_release_mu is non-zero. Other bits are set or - cleared according to whether we woke any threads, - whether any waiters remain, and whether any of them - are writers. */ - old_word = ATM_LOAD (&mu->word); - while (!ATM_CAS_REL (&mu->word, old_word, - ((old_word-late_release_mu)|set_on_release) & - ~clear_on_release)) { /* release CAS */ - old_word = ATM_LOAD (&mu->word); - } - /* Wake the waiters. */ - for (p = nsync_dll_first_ (wake); p != NULL; p = next) { - next = nsync_dll_next_ (wake, p); - wake = nsync_dll_remove_ (wake, p); - ATM_STORE_REL (&DLL_NSYNC_WAITER (p)->waiting, 0); - nsync_mu_semaphore_v (&DLL_WAITER (p)->sem); - } - return; - } - attempts = nsync_spin_delay_ (attempts); - } -} - -/* Unlock *mu, which must be held in write mode, and wake waiters, if appropriate. */ -void nsync_mu_unlock (nsync_mu *mu) { - IGNORE_RACES_START (); - /* C is not a garbage-collected language, so we cannot release until we - can be sure that we will not have to touch the mutex again to wake a - waiter. Another thread could acquire, decrement a reference count - and deallocate the mutex before the current thread touched the mutex - word again. */ - if (!ATM_CAS_REL (&mu->word, MU_WLOCK, 0)) { - uint32_t old_word = ATM_LOAD (&mu->word); - /* Clear MU_ALL_FALSE because the critical section we're just - leaving may have made some conditions true. */ - uint32_t new_word = (old_word - MU_WLOCK) & ~MU_ALL_FALSE; - /* Sanity check: mutex must be held in write mode, and there - must be no readers. */ - if ((new_word & (MU_RLOCK_FIELD | MU_WLOCK)) != 0) { - if ((old_word & MU_RLOCK_FIELD) != 0) { - nsync_panic_ ("attempt to nsync_mu_unlock() an nsync_mu " - "held in read mode\n"); - } else { - nsync_panic_ ("attempt to nsync_mu_unlock() an nsync_mu " - "not held in write mode\n"); - } - } else if ((old_word & (MU_WAITING|MU_DESIG_WAKER)) == MU_WAITING || - !ATM_CAS_REL (&mu->word, old_word, new_word)) { - /* There are waiters and no designated waker, or - our initial CAS attempt failed, to use slow path. */ - nsync_mu_unlock_slow_ (mu, nsync_writer_type_); - } - } - IGNORE_RACES_END (); -} - -/* Unlock *mu, which must be held in read mode, and wake waiters, if appropriate. */ -void nsync_mu_runlock (nsync_mu *mu) { - IGNORE_RACES_START (); - /* See comment in nsync_mu_unlock(). */ - if (!ATM_CAS_REL (&mu->word, MU_RLOCK, 0)) { - uint32_t old_word = ATM_LOAD (&mu->word); - /* Sanity check: mutex must not be held in write mode and - reader count must not be 0. */ - if (((old_word ^ MU_WLOCK) & (MU_WLOCK | MU_RLOCK_FIELD)) == 0) { - if ((old_word & MU_WLOCK) != 0) { - nsync_panic_ ("attempt to nsync_mu_runlock() an nsync_mu " - "held in write mode\n"); - } else { - nsync_panic_ ("attempt to nsync_mu_runlock() an nsync_mu " - "not held in read mode\n"); - } - } else if ((old_word & (MU_WAITING | MU_DESIG_WAKER)) == MU_WAITING && - (old_word & (MU_RLOCK_FIELD|MU_ALL_FALSE)) == MU_RLOCK) { - /* There are waiters and no designated waker, the last - reader is unlocking, and not all waiters have a - false condition. So we must take the slow path to - attempt to wake a waiter. */ - nsync_mu_unlock_slow_ (mu, nsync_reader_type_); - } else if (!ATM_CAS_REL (&mu->word, old_word, old_word - MU_RLOCK)) { - /* CAS attempt failed, so take slow path. */ - nsync_mu_unlock_slow_ (mu, nsync_reader_type_); - } - } - IGNORE_RACES_END (); -} - -/* Abort if *mu is not held in write mode. */ -void nsync_mu_assert_held (const nsync_mu *mu) { - IGNORE_RACES_START (); - if ((ATM_LOAD (&mu->word) & MU_WHELD_IF_NON_ZERO) == 0) { - nsync_panic_ ("nsync_mu not held in write mode\n"); - } - IGNORE_RACES_END (); -} - -/* Abort if *mu is not held in read or write mode. */ -void nsync_mu_rassert_held (const nsync_mu *mu) { - IGNORE_RACES_START (); - if ((ATM_LOAD (&mu->word) & MU_ANY_LOCK) == 0) { - nsync_panic_ ("nsync_mu not held in some mode\n"); - } - IGNORE_RACES_END (); -} - -/* Return whether *mu is held in read mode. - Requires that *mu is held in some mode. */ -int nsync_mu_is_reader (const nsync_mu *mu) { - uint32_t word; - IGNORE_RACES_START (); - word = ATM_LOAD (&mu->word); - if ((word & MU_ANY_LOCK) == 0) { - nsync_panic_ ("nsync_mu not held in some mode\n"); - } - IGNORE_RACES_END (); - return ((word & MU_WLOCK) == 0); -} diff --git a/third_party/nsync/mu.h b/third_party/nsync/mu.h deleted file mode 100644 index 8d72fea6c..000000000 --- a/third_party/nsync/mu.h +++ /dev/null @@ -1,103 +0,0 @@ -#ifndef NSYNC_MU_H_ -#define NSYNC_MU_H_ -#include "third_party/nsync/atomic.h" -#if !(__ASSEMBLER__ + __LINKER__ + 0) -COSMOPOLITAN_C_START_ - -struct nsync_dll_element_s_; - -/* An nsync_mu is a lock. If initialized to zero, it's valid and unlocked. - - An nsync_mu can be "free", held by a single thread (aka fiber, - goroutine) in "write" (exclusive) mode, or by many threads in "read" - (shared) mode. A thread that acquires it should eventually release - it. It is illegal to acquire an nsync_mu in one thread and release it - in another. It is illegal for a thread to reacquire an nsync_mu while - holding it (even a second share of a "read" lock). - - Example usage: - - static struct foo { - nsync_mu mu; // protects invariant a+b==0 on fields below. - int a; - int b; - } p = { NSYNC_MU_INIT, 0, 0 }; - // .... - nsync_mu_lock (&p.mu); - // The current thread now has exclusive access to p.a and p.b; - // invariant assumed true. - p.a++; - p.b--; // restore invariant p.a+p.b==0 before releasing p.mu - nsync_mu_unlock (&p.mu) - - Mutexes can be used with condition variables; see nsync_cv.h. - - nsync_mu_wait() and nsync_mu_wait_with_deadline() can be used instead - of condition variables. See nsync_mu_wait.h for more details. Example - use of nsync_mu_wait() to wait for p.a==0, using definition above: - - int a_is_zero (const void *condition_arg) { - return (((const struct foo *)condition_arg)->a == 0); - } - ... - nsync_mu_lock (&p.mu); - nsync_mu_wait (&p.mu, &a_is_zero, &p, NULL); - // The current thread now has exclusive access to - // p.a and p.b, and p.a==0. - ... - nsync_mu_unlock (&p.mu); - -*/ -typedef struct nsync_mu_s_ { - nsync_atomic_uint32_ word; /* internal use only */ - struct nsync_dll_element_s_ *waiters; /* internal use only */ -} nsync_mu; - -/* An nsync_mu should be zeroed to initialize, which can be accomplished - by initializing with static initializer NSYNC_MU_INIT, or by setting - the entire structure to all zeroes, or using nsync_mu_init(). */ -#define NSYNC_MU_INIT \ - { NSYNC_ATOMIC_UINT32_INIT_, 0 } -void nsync_mu_init(nsync_mu *mu); - -/* Block until *mu is free and then acquire it in writer mode. Requires - that the calling thread not already hold *mu in any mode. */ -void nsync_mu_lock(nsync_mu *mu); - -/* Unlock *mu, which must have been acquired in write mode by the - calling thread, and wake waiters, if appropriate. */ -void nsync_mu_unlock(nsync_mu *mu); - -/* Attempt to acquire *mu in writer mode without blocking, and return - non-zero iff successful. Return non-zero with high probability if *mu - was free on entry. */ -int nsync_mu_trylock(nsync_mu *mu); - -/* Block until *mu can be acquired in reader mode and then acquire it. - Requires that the calling thread not already hold *mu in any mode. */ -void nsync_mu_rlock(nsync_mu *mu); - -/* Unlock *mu, which must have been acquired in read mode by the calling - thread, and wake waiters, if appropriate. */ -void nsync_mu_runlock(nsync_mu *mu); - -/* Attempt to acquire *mu in reader mode without blocking, and return - non-zero iff successful. Return non-zero with high probability if *mu - was free on entry. Perhaps fail to acquire if a writer is waiting, to - avoid starvation. */ -int nsync_mu_rtrylock(nsync_mu *mu); - -/* May abort if *mu is not held in write mode by the calling thread. */ -void nsync_mu_assert_held(const nsync_mu *mu); - -/* May abort if *mu is not held in read or write mode - by the calling thread. */ -void nsync_mu_rassert_held(const nsync_mu *mu); - -/* Return whether *mu is held in read mode. - Requires that the calling thread holds *mu in some mode. */ -int nsync_mu_is_reader(const nsync_mu *mu); - -COSMOPOLITAN_C_END_ -#endif /* !(__ASSEMBLER__ + __LINKER__ + 0) */ -#endif /* NSYNC_MU_H_ */ diff --git a/third_party/nsync/mu_semaphore.c b/third_party/nsync/mu_semaphore.c deleted file mode 100644 index a30ef4873..000000000 --- a/third_party/nsync/mu_semaphore.c +++ /dev/null @@ -1,56 +0,0 @@ -/*-*- mode:c;indent-tabs-mode:t;c-basic-offset:8;tab-width:8;coding:utf-8 -*-│ -│vi: set et ft=c ts=8 tw=8 fenc=utf-8 :vi│ -╞══════════════════════════════════════════════════════════════════════════════╡ -│ Copyright 2022 Justine Alexandra Roberts Tunney │ -│ │ -│ Permission to use, copy, modify, and/or distribute this software for │ -│ any purpose with or without fee is hereby granted, provided that the │ -│ above copyright notice and this permission notice appear in all copies. │ -│ │ -│ THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL │ -│ WARRANTIES WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED │ -│ WARRANTIES OF MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE │ -│ AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL │ -│ DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR │ -│ PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER │ -│ TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR │ -│ PERFORMANCE OF THIS SOFTWARE. │ -╚─────────────────────────────────────────────────────────────────────────────*/ -#include "libc/dce.h" -#include "third_party/nsync/mu_semaphore.h" -#include "third_party/nsync/mu_semaphore.internal.h" -// clang-format off - -/* Initialize *s; the initial value is 0. */ -void nsync_mu_semaphore_init (nsync_semaphore *s) { - if (!IsWindows ()) - nsync_mu_semaphore_init_futex (s); - else - nsync_mu_semaphore_init_win32 (s); -} - -/* Wait until the count of *s exceeds 0, and decrement it. */ -void nsync_mu_semaphore_p (nsync_semaphore *s) { - if (!IsWindows ()) - nsync_mu_semaphore_p_futex (s); - else - nsync_mu_semaphore_p_win32 (s); -} - -/* Wait until one of: - the count of *s is non-zero, in which case decrement *s and return 0; - or abs_deadline expires, in which case return ETIMEDOUT. */ -int nsync_mu_semaphore_p_with_deadline (nsync_semaphore *s, nsync_time abs_deadline) { - if (!IsWindows ()) - return nsync_mu_semaphore_p_with_deadline_futex (s, abs_deadline); - else - return nsync_mu_semaphore_p_with_deadline_win32 (s, abs_deadline); -} - -/* Ensure that the count of *s is at least 1. */ -void nsync_mu_semaphore_v (nsync_semaphore *s) { - if (!IsWindows ()) - nsync_mu_semaphore_v_futex (s); - else - nsync_mu_semaphore_v_win32 (s); -} diff --git a/third_party/nsync/mu_semaphore.h b/third_party/nsync/mu_semaphore.h deleted file mode 100644 index d57e1b770..000000000 --- a/third_party/nsync/mu_semaphore.h +++ /dev/null @@ -1,28 +0,0 @@ -#ifndef NSYNC_SEM_H_ -#define NSYNC_SEM_H_ -#include "third_party/nsync/time.h" -#if !(__ASSEMBLER__ + __LINKER__ + 0) -COSMOPOLITAN_C_START_ - -typedef struct nsync_semaphore_s_ { - void *sem_space[32]; /* space used by implementation */ -} nsync_semaphore; - -/* Initialize *s; the initial value is 0. */ -void nsync_mu_semaphore_init(nsync_semaphore *s); - -/* Wait until the count of *s exceeds 0, and decrement it. */ -void nsync_mu_semaphore_p(nsync_semaphore *s); - -/* Wait until one of: the count of *s is non-zero, in which case - decrement *s and return 0; or abs_deadline expires, in which case - return ETIMEDOUT. */ -int nsync_mu_semaphore_p_with_deadline(nsync_semaphore *s, - nsync_time abs_deadline); - -/* Ensure that the count of *s is at least 1. */ -void nsync_mu_semaphore_v(nsync_semaphore *s); - -COSMOPOLITAN_C_END_ -#endif /* !(__ASSEMBLER__ + __LINKER__ + 0) */ -#endif /* NSYNC_SEM_H_ */ diff --git a/third_party/nsync/mu_semaphore.internal.h b/third_party/nsync/mu_semaphore.internal.h deleted file mode 100644 index 8fac00aa0..000000000 --- a/third_party/nsync/mu_semaphore.internal.h +++ /dev/null @@ -1,20 +0,0 @@ -#ifndef NSYNC_MU_SEMAPHORE_INTERNAL_H_ -#define NSYNC_MU_SEMAPHORE_INTERNAL_H_ -#include "third_party/nsync/mu_semaphore.h" -#include "third_party/nsync/time.h" -#if !(__ASSEMBLER__ + __LINKER__ + 0) -COSMOPOLITAN_C_START_ - -void nsync_mu_semaphore_init_futex(nsync_semaphore *); -void nsync_mu_semaphore_p_futex(nsync_semaphore *); -int nsync_mu_semaphore_p_with_deadline_futex(nsync_semaphore *, nsync_time); -void nsync_mu_semaphore_v_futex(nsync_semaphore *); - -void nsync_mu_semaphore_init_win32(nsync_semaphore *); -void nsync_mu_semaphore_p_win32(nsync_semaphore *); -int nsync_mu_semaphore_p_with_deadline_win32(nsync_semaphore *, nsync_time); -void nsync_mu_semaphore_v_win32(nsync_semaphore *); - -COSMOPOLITAN_C_END_ -#endif /* !(__ASSEMBLER__ + __LINKER__ + 0) */ -#endif /* NSYNC_MU_SEMAPHORE_INTERNAL_H_ */ diff --git a/third_party/nsync/mu_semaphore_futex.c b/third_party/nsync/mu_semaphore_futex.c deleted file mode 100644 index 237f83ed2..000000000 --- a/third_party/nsync/mu_semaphore_futex.c +++ /dev/null @@ -1,124 +0,0 @@ -/*-*- mode:c;indent-tabs-mode:t;c-basic-offset:8;tab-width:8;coding:utf-8 -*-│ -│vi: set et ft=c ts=8 tw=8 fenc=utf-8 :vi│ -╞══════════════════════════════════════════════════════════════════════════════╡ -│ Copyright 2016 Google Inc. │ -│ │ -│ Licensed under the Apache License, Version 2.0 (the "License"); │ -│ you may not use this file except in compliance with the License. │ -│ You may obtain a copy of the License at │ -│ │ -│ http://www.apache.org/licenses/LICENSE-2.0 │ -│ │ -│ Unless required by applicable law or agreed to in writing, software │ -│ distributed under the License is distributed on an "AS IS" BASIS, │ -│ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. │ -│ See the License for the specific language governing permissions and │ -│ limitations under the License. │ -╚─────────────────────────────────────────────────────────────────────────────*/ -#include "libc/errno.h" -#include "libc/str/str.h" -#include "libc/thread/thread.h" -#include "third_party/nsync/atomic.h" -#include "third_party/nsync/atomic.internal.h" -#include "third_party/nsync/futex.internal.h" -#include "third_party/nsync/mu_semaphore.h" -#include "third_party/nsync/mu_semaphore.internal.h" - -asm(".ident\t\"\\n\\n\ -*NSYNC (Apache 2.0)\\n\ -Copyright 2016 Google, Inc.\\n\ -https://github.com/google/nsync\""); -// clang-format off - -/* Check that atomic operations on nsync_atomic_uint32_ can be applied to int. */ -static const int assert_int_size = 1 / - (sizeof (assert_int_size) == sizeof (uint32_t) && - sizeof (nsync_atomic_uint32_) == sizeof (uint32_t)); - -#define ASSERT(x) do { if (!(x)) { *(volatile int *)0 = 0; } } while (0) - -struct futex { - int i; /* lo half=count; hi half=waiter count */ -}; - -static nsync_semaphore *sem_big_enough_for_futex = (nsync_semaphore *) (uintptr_t)(1 / - (sizeof (struct futex) <= sizeof (*sem_big_enough_for_futex))); - -/* Initialize *s; the initial value is 0. */ -void nsync_mu_semaphore_init_futex (nsync_semaphore *s) { - struct futex *f = (struct futex *) s; - f->i = 0; -} - -/* Wait until the count of *s exceeds 0, and decrement it. */ -void nsync_mu_semaphore_p_futex (nsync_semaphore *s) { - struct futex *f = (struct futex *) s; - int i; - do { - i = ATM_LOAD ((nsync_atomic_uint32_ *) &f->i); - if (i == 0) { - int futex_result = nsync_futex_wait_ (&f->i, i, PTHREAD_PROCESS_PRIVATE, NULL); - ASSERT (futex_result == 0 || - futex_result == -EINTR || - futex_result == -EWOULDBLOCK); - } - } while (i == 0 || !ATM_CAS_ACQ ((nsync_atomic_uint32_ *) &f->i, i, i-1)); -} - -/* Wait until one of: - the count of *s is non-zero, in which case decrement *s and return 0; - or abs_deadline expires, in which case return ETIMEDOUT. */ -int nsync_mu_semaphore_p_with_deadline_futex (nsync_semaphore *s, nsync_time abs_deadline) { - struct futex *f = (struct futex *)s; - int i; - int result = 0; - do { - i = ATM_LOAD ((nsync_atomic_uint32_ *) &f->i); - if (i == 0) { - int futex_result; - struct timespec ts_buf; - const struct timespec *ts = NULL; - if (nsync_time_cmp (abs_deadline, nsync_time_no_deadline) != 0) { - memset (&ts_buf, 0, sizeof (ts_buf)); - if (FUTEX_TIMEOUT_IS_ABSOLUTE) { - ts_buf.tv_sec = NSYNC_TIME_SEC (abs_deadline); - ts_buf.tv_nsec = NSYNC_TIME_NSEC (abs_deadline); - } else { - nsync_time now; - now = nsync_time_now (); - if (nsync_time_cmp (now, abs_deadline) > 0) { - ts_buf.tv_sec = 0; - ts_buf.tv_nsec = 0; - } else { - nsync_time rel_deadline; - rel_deadline = nsync_time_sub (abs_deadline, now); - ts_buf.tv_sec = NSYNC_TIME_SEC (rel_deadline); - ts_buf.tv_nsec = NSYNC_TIME_NSEC (rel_deadline); - } - } - ts = &ts_buf; - } - futex_result = nsync_futex_wait_ (&f->i, i, PTHREAD_PROCESS_PRIVATE, ts); - ASSERT (futex_result == 0 || - futex_result == -EINTR || - futex_result == -ETIMEDOUT || - futex_result == -EWOULDBLOCK); - /* Some systems don't wait as long as they are told. */ - if (futex_result == -ETIMEDOUT && - nsync_time_cmp (abs_deadline, nsync_time_now ()) <= 0) { - result = ETIMEDOUT; - } - } - } while (result == 0 && (i == 0 || !ATM_CAS_ACQ ((nsync_atomic_uint32_ *) &f->i, i, i - 1))); - return (result); -} - -/* Ensure that the count of *s is at least 1. */ -void nsync_mu_semaphore_v_futex (nsync_semaphore *s) { - struct futex *f = (struct futex *) s; - uint32_t old_value; - do { - old_value = ATM_LOAD ((nsync_atomic_uint32_ *) &f->i); - } while (!ATM_CAS_REL ((nsync_atomic_uint32_ *) &f->i, old_value, old_value+1)); - ASSERT (nsync_futex_wake_ (&f->i, 1, PTHREAD_PROCESS_PRIVATE) >= 0); -} diff --git a/third_party/nsync/mu_semaphore_win32.c b/third_party/nsync/mu_semaphore_win32.c deleted file mode 100644 index 9dc8e2927..000000000 --- a/third_party/nsync/mu_semaphore_win32.c +++ /dev/null @@ -1,84 +0,0 @@ -/*-*- mode:c;indent-tabs-mode:t;c-basic-offset:8;tab-width:8;coding:utf-8 -*-│ -│vi: set et ft=c ts=8 tw=8 fenc=utf-8 :vi│ -╞══════════════════════════════════════════════════════════════════════════════╡ -│ Copyright 2016 Google Inc. │ -│ │ -│ Licensed under the Apache License, Version 2.0 (the "License"); │ -│ you may not use this file except in compliance with the License. │ -│ You may obtain a copy of the License at │ -│ │ -│ http://www.apache.org/licenses/LICENSE-2.0 │ -│ │ -│ Unless required by applicable law or agreed to in writing, software │ -│ distributed under the License is distributed on an "AS IS" BASIS, │ -│ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. │ -│ See the License for the specific language governing permissions and │ -│ limitations under the License. │ -╚─────────────────────────────────────────────────────────────────────────────*/ -#include "libc/errno.h" -#include "libc/nt/enum/wait.h" -#include "libc/nt/synchronization.h" -#include "libc/runtime/runtime.h" -#include "third_party/nsync/mu_semaphore.h" -#include "third_party/nsync/mu_semaphore.internal.h" -#include "third_party/nsync/time.h" - -asm(".ident\t\"\\n\\n\ -*NSYNC (Apache 2.0)\\n\ -Copyright 2016 Google, Inc.\\n\ -https://github.com/google/nsync\""); -// clang-format off - -/* Initialize *s; the initial value is 0. */ -void nsync_mu_semaphore_init_win32 (nsync_semaphore *s) { - int64_t *h = (int64_t *) s; - *h = CreateSemaphore(NULL, 0, 1, NULL); - if (!*h) notpossible; -} - -/* Wait until the count of *s exceeds 0, and decrement it. */ -void nsync_mu_semaphore_p_win32 (nsync_semaphore *s) { - int64_t *h = (int64_t *) s; - WaitForSingleObject(*h, -1u); -} - -/* Wait until one of: - the count of *s is non-zero, in which case decrement *s and return 0; - or abs_deadline expires, in which case return ETIMEDOUT. */ -int nsync_mu_semaphore_p_with_deadline_win32 (nsync_semaphore *s, nsync_time abs_deadline) { - int64_t *h = (int64_t *) s; - int result; - - if (nsync_time_cmp (abs_deadline, nsync_time_no_deadline) == 0) { - result = WaitForSingleObject(*h, -1u); - } else { - nsync_time now; - now = nsync_time_now (); - do { - if (nsync_time_cmp (abs_deadline, now) <= 0) { - result = WaitForSingleObject (*h, 0); - } else { - nsync_time delay; - delay = nsync_time_sub (abs_deadline, now); - if (NSYNC_TIME_SEC (delay) > 1000*1000) { - result = WaitForSingleObject (*h, 1000*1000); - } else { - result = WaitForSingleObject (*h, - (unsigned) (NSYNC_TIME_SEC (delay) * 1000 + - (NSYNC_TIME_NSEC (delay) + 999999) / (1000 * 1000))); - } - } - if (result == kNtWaitTimeout) { - now = nsync_time_now (); - } - } while (result == kNtWaitTimeout && /* Windows generates early wakeups. */ - nsync_time_cmp (abs_deadline, now) > 0); - } - return (result == kNtWaitTimeout ? ETIMEDOUT : 0); -} - -/* Ensure that the count of *s is at least 1. */ -void nsync_mu_semaphore_v_win32 (nsync_semaphore *s) { - int64_t *h = (int64_t *) s; - ReleaseSemaphore(*h, 1, NULL); -} diff --git a/third_party/nsync/mu_wait.h b/third_party/nsync/mu_wait.h deleted file mode 100644 index 7a5a3478b..000000000 --- a/third_party/nsync/mu_wait.h +++ /dev/null @@ -1,118 +0,0 @@ -#ifndef NSYNC_MU_WAIT_H_ -#define NSYNC_MU_WAIT_H_ -#include "third_party/nsync/mu.h" -#include "third_party/nsync/time.h" -#if !(__ASSEMBLER__ + __LINKER__ + 0) -COSMOPOLITAN_C_START_ - -/* nsync_mu_wait() and nsync_mu_wait_with_deadline() can be used instead - of condition variables. In many straightforward situations they are - of equivalent performance and are somewhat easier to use, because - unlike condition variables, they do not require that the waits be - placed in a loop, and they do not require explicit wakeup calls. - Example: - - Definitions: - - static nsync_mu mu = NSYNC_MU_INIT; - static int i = 0; // protected by mu - // Condition for use with nsync_mu_wait(). - static int int_is_zero (const void *v) { - return (*(const int *)v == 0); - } - - Waiter: - - nsync_mu_lock (&mu); - // Wait until i is zero. - nsync_mu_wait (&mu, &int_is_zero, &i, NULL); - // i is known to be zero here. - // ... - nsync_mu_unlock (&mu); - - Thread potentially making i zero: - - nsync_mu_lock (&mu); - i--; - // No need to signal that i may have become zero. The unlock call - // below will evaluate waiters' conditions to decide which to wake. - nsync_mu_unlock (&mu); - - It is legal to use conditional critical sections and condition - variables on the same mutex. - - -------------- - - The implementation benefits from determining whether waiters are - waiting for the same condition; it may then evaluate a condition once - on behalf of several waiters. Two waiters have equal condition if - their "condition" pointers are equal, and either: - - - their "condition_arg" pointers are equal, or - - - "condition_arg_eq" is non-null and (*condition_arg_eq) - (condition_arg0, condition_arg1) returns non-zero. - - *condition_arg_eq will not be invoked unless the "condition" pointers - are equal, and the "condition_arg" pointers are unequal. - - If many waiters wait for distinct conditions simultaneously, - condition variables may be faster. - */ - -struct nsync_note_s_; /* forward declaration for an nsync_note */ - -/* Return when (*condition) (condition_arg) is true. Perhaps unlock and - relock *mu while blocked waiting for the condition to become true. - nsync_mu_wait() is equivalent to nsync_mu_wait_with_deadline() with - abs_deadline==nsync_time_no_deadline, and cancel_note==NULL. - - Requires that *mu be held on entry. See nsync_mu_wait_with_deadline() - for more details on *condition and *condition_arg_eq. */ -void nsync_mu_wait(nsync_mu *mu, int (*condition)(const void *condition_arg), - const void *condition_arg, - int (*condition_arg_eq)(const void *a, const void *b)); - -/* Return when at least one of: (*condition) (condition_arg) is true, - the deadline expires, or *cancel_note is notified. Perhaps unlock and - relock *mu while blocked waiting for one of these events, but always - return with *mu held. Return 0 iff the (*condition) (condition_arg) - is true on return, and otherwise either ETIMEDOUT or ECANCELED, - depending on why the call returned early. Callers should use - abs_deadline==nsync_time_no_deadline for no deadline, and - cancel_note==NULL for no cancellation. - - Requires that *mu be held on entry. - - The implementation may call *condition from any thread using the - mutex, and while holding *mu in either read or write mode; it - guarantees that any thread calling *condition will hold *mu in some - mode. Requires that (*condition) (condition_arg) neither modify state - protected by *mu, nor return a value dependent on state not protected - by *mu. To depend on time, use the abs_deadline parameter. - (Conventional use of condition variables have the same restrictions - on the conditions tested by the while-loop.) If non-null, - condition_arg_eq should return whether two condition_arg calls with - the same "condition" pointer are considered equivalent; it should - have no side-effects. */ -int nsync_mu_wait_with_deadline( - nsync_mu *mu, int (*condition)(const void *condition_arg), - const void *condition_arg, - int (*condition_arg_eq)(const void *a, const void *b), - nsync_time abs_deadline, struct nsync_note_s_ *cancel_note); - -/* Unlock *mu, which must be held in write mode, and wake waiters, if - appropriate. Unlike nsync_mu_unlock(), this call is not required to - wake nsync_mu_wait/nsync_mu_wait_with_deadline calls on conditions - that were false before this thread acquired the lock. This call - should be used only at the end of critical sections for which: - - nsync_mu_wait and/or nsync_mu_wait_with_deadline are in use on the same - mutex, - - this critical section cannot make the condition true for any of those - nsync_mu_wait/nsync_mu_wait_with_deadline waits, and - - when performance is significantly improved by using this call. */ -void nsync_mu_unlock_without_wakeup(nsync_mu *mu); - -COSMOPOLITAN_C_END_ -#endif /* !(__ASSEMBLER__ + __LINKER__ + 0) */ -#endif /* NSYNC_MU_WAIT_H_ */ diff --git a/third_party/nsync/note.h b/third_party/nsync/note.h deleted file mode 100644 index 213a2d70a..000000000 --- a/third_party/nsync/note.h +++ /dev/null @@ -1,51 +0,0 @@ -#ifndef NSYNC_NOTE_H_ -#define NSYNC_NOTE_H_ -#include "third_party/nsync/time.h" -#if !(__ASSEMBLER__ + __LINKER__ + 0) -COSMOPOLITAN_C_START_ - -/* An nsync_note represents a single bit that can transition from 0 to 1 - at most once. When 1, the note is said to be notified. There are - operations to wait for the transition, which can be triggered either - by an explicit call, or timer expiry. Notes can have parent notes; a - note becomes notified if its parent becomes notified. */ -typedef struct nsync_note_s_ *nsync_note; - -/* Return a freshly allocated nsync_note, or NULL if an nsync_note - cannot be created. - - If parent!=NULL, the allocated nsync_note's parent will be parent. - The newaly allocated note will be automatically notified at - abs_deadline, and is notified at initialization if - abs_deadline==nsync_zero_time. - - nsync_notes should be passed to nsync_note_free() when no longer needed. */ -nsync_note nsync_note_new(nsync_note parent, nsync_time abs_deadline); - -/* Free resources associated with n. Requires that n was allocated by - nsync_note_new(), and no concurrent or future operations are applied - to n directly. - - It is legal to call nsync_note_free() on a node even if it has a - parent or children that are in use; if n has both a parent and - children, n's parent adopts its children. */ -void nsync_note_free(nsync_note n); - -/* Notify n and all its descendants. */ -void nsync_note_notify(nsync_note n); - -/* Return whether n has been notified. */ -int nsync_note_is_notified(nsync_note n); - -/* Wait until n has been notified or abs_deadline is reached, and return - whether n has been notified. If abs_deadline==nsync_time_no_deadline, - the deadline is far in the future. */ -int nsync_note_wait(nsync_note n, nsync_time abs_deadline); - -/* Return the expiry time associated with n. This is the minimum of the - abs_deadline passed on creation and that of any of its ancestors. */ -nsync_time nsync_note_expiry(nsync_note n); - -COSMOPOLITAN_C_END_ -#endif /* !(__ASSEMBLER__ + __LINKER__ + 0) */ -#endif /* NSYNC_NOTE_H_ */ diff --git a/third_party/nsync/nsync.mk b/third_party/nsync/nsync.mk deleted file mode 100644 index 89e03a84a..000000000 --- a/third_party/nsync/nsync.mk +++ /dev/null @@ -1,58 +0,0 @@ -#-*-mode:makefile-gmake;indent-tabs-mode:t;tab-width:8;coding:utf-8-*-┐ -#───vi: set et ft=make ts=8 tw=8 fenc=utf-8 :vi───────────────────────┘ - -PKGS += THIRD_PARTY_NSYNC - -THIRD_PARTY_NSYNC_SRCS = $(THIRD_PARTY_NSYNC_A_SRCS) -THIRD_PARTY_NSYNC_HDRS = $(THIRD_PARTY_NSYNC_A_HDRS) - -THIRD_PARTY_NSYNC_ARTIFACTS += THIRD_PARTY_NSYNC_A -THIRD_PARTY_NSYNC = $(THIRD_PARTY_NSYNC_A_DEPS) $(THIRD_PARTY_NSYNC_A) -THIRD_PARTY_NSYNC_A = o/$(MODE)/third_party/nsync/nsync.a -THIRD_PARTY_NSYNC_A_FILES := $(wildcard third_party/nsync/*) -THIRD_PARTY_NSYNC_A_HDRS = $(filter %.h,$(THIRD_PARTY_NSYNC_A_FILES)) -THIRD_PARTY_NSYNC_A_SRCS = $(filter %.c,$(THIRD_PARTY_NSYNC_A_FILES)) - -THIRD_PARTY_NSYNC_A_OBJS = \ - $(THIRD_PARTY_NSYNC_A_SRCS:%.c=o/$(MODE)/%.o) - -THIRD_PARTY_NSYNC_A_DIRECTDEPS = \ - LIBC_CALLS \ - LIBC_INTRIN \ - LIBC_NEXGEN32E \ - LIBC_NT_KERNEL32 \ - LIBC_STR \ - LIBC_STUBS \ - LIBC_SYSV \ - LIBC_SYSV_CALLS - -THIRD_PARTY_NSYNC_A_DEPS := \ - $(call uniq,$(foreach x,$(THIRD_PARTY_NSYNC_A_DIRECTDEPS),$($(x)))) - -THIRD_PARTY_NSYNC_A_CHECKS = \ - $(THIRD_PARTY_NSYNC_A).pkg \ - $(THIRD_PARTY_NSYNC_A_HDRS:%=o/$(MODE)/%.ok) - -$(THIRD_PARTY_NSYNC_A): \ - third_party/nsync/ \ - $(THIRD_PARTY_NSYNC_A).pkg \ - $(THIRD_PARTY_NSYNC_A_OBJS) - -$(THIRD_PARTY_NSYNC_A).pkg: \ - $(THIRD_PARTY_NSYNC_A_OBJS) \ - $(foreach x,$(THIRD_PARTY_NSYNC_A_DIRECTDEPS),$($(x)_A).pkg) - -$(THIRD_PARTY_NSYNC_A_OBJS): private \ - OVERRIDE_CCFLAGS += \ - -ffunction-sections \ - -fdata-sections - -THIRD_PARTY_NSYNC_LIBS = $(foreach x,$(THIRD_PARTY_NSYNC_ARTIFACTS),$($(x))) -THIRD_PARTY_NSYNC_SRCS = $(foreach x,$(THIRD_PARTY_NSYNC_ARTIFACTS),$($(x)_SRCS)) -THIRD_PARTY_NSYNC_CHECKS = $(foreach x,$(THIRD_PARTY_NSYNC_ARTIFACTS),$($(x)_CHECKS)) -THIRD_PARTY_NSYNC_OBJS = $(foreach x,$(THIRD_PARTY_NSYNC_ARTIFACTS),$($(x)_OBJS)) -$(THIRD_PARTY_NSYNC_OBJS): third_party/nsync/nsync.mk - -.PHONY: o/$(MODE)/third_party/nsync -o/$(MODE)/third_party/nsync: $(THIRD_PARTY_NSYNC_CHECKS) - diff --git a/third_party/nsync/once.h b/third_party/nsync/once.h deleted file mode 100644 index 4144df129..000000000 --- a/third_party/nsync/once.h +++ /dev/null @@ -1,37 +0,0 @@ -#ifndef NSYNC_ONCE_H_ -#define NSYNC_ONCE_H_ -#include "third_party/nsync/atomic.h" -#if !(__ASSEMBLER__ + __LINKER__ + 0) -COSMOPOLITAN_C_START_ - -/* An nsync_once allows a function to be called exactly once, when first - referenced. */ -typedef nsync_atomic_uint32_ nsync_once; - -/* An initializer for nsync_once; it is guaranteed to be all zeroes. */ -#define NSYNC_ONCE_INIT NSYNC_ATOMIC_UINT32_INIT_ - -/* The first time nsync_run_once() or nsync_run_once_arg() is applied to - *once, the supplied function is run (with argument, in the case of - nsync_run_once_arg()). Other callers will wait until the run of the - function is complete, and then return without running the function - again. */ -void nsync_run_once(nsync_once *once, void (*f)(void)); -void nsync_run_once_arg(nsync_once *once, void (*farg)(void *arg), void *arg); - -/* Same as nsync_run_once()/nsync_run_once_arg() but uses a spinloop. - Can be used on the same nsync_once as - nsync_run_once/nsync_run_once_arg(). - - These *_spin variants should be used only in contexts where normal - blocking is disallowed, such as within user-space schedulers, when - the runtime is not fully initialized, etc. They provide no - significant performance benefit, and they should be avoided in normal - code. */ -void nsync_run_once_spin(nsync_once *once, void (*f)(void)); -void nsync_run_once_arg_spin(nsync_once *once, void (*farg)(void *arg), - void *arg); - -COSMOPOLITAN_C_END_ -#endif /* !(__ASSEMBLER__ + __LINKER__ + 0) */ -#endif /* NSYNC_ONCE_H_ */ diff --git a/third_party/nsync/panic.c b/third_party/nsync/panic.c deleted file mode 100644 index 6606ab06e..000000000 --- a/third_party/nsync/panic.c +++ /dev/null @@ -1,30 +0,0 @@ -/*-*- mode:c;indent-tabs-mode:t;c-basic-offset:8;tab-width:8;coding:utf-8 -*-│ -│vi: set et ft=c ts=8 tw=8 fenc=utf-8 :vi│ -╞══════════════════════════════════════════════════════════════════════════════╡ -│ Copyright 2022 Justine Alexandra Roberts Tunney │ -│ │ -│ Permission to use, copy, modify, and/or distribute this software for │ -│ any purpose with or without fee is hereby granted, provided that the │ -│ above copyright notice and this permission notice appear in all copies. │ -│ │ -│ THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL │ -│ WARRANTIES WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED │ -│ WARRANTIES OF MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE │ -│ AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL │ -│ DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR │ -│ PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER │ -│ TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR │ -│ PERFORMANCE OF THIS SOFTWARE. │ -╚─────────────────────────────────────────────────────────────────────────────*/ -#include "libc/calls/calls.h" -#include "third_party/nsync/common.internal.h" -// clang-format off - -/* Aborts after printing the nul-terminated string s[]. */ -void nsync_panic_ (const char *s) { - size_t n = 0; - while (s[n]) ++n; - write (2, "panic: ", 7); - write (2, s, n); - notpossible; -} diff --git a/third_party/nsync/races.internal.h b/third_party/nsync/races.internal.h deleted file mode 100644 index 84ff665f5..000000000 --- a/third_party/nsync/races.internal.h +++ /dev/null @@ -1,36 +0,0 @@ -#ifndef COSMOPOLITAN_THIRD_PARTY_NSYNC_RACES_INTERNAL_H_ -#define COSMOPOLITAN_THIRD_PARTY_NSYNC_RACES_INTERNAL_H_ -#if !(__ASSEMBLER__ + __LINKER__ + 0) -COSMOPOLITAN_C_START_ - -/* Annotations for race detectors. */ -#if defined(__has_feature) && !defined(__SANITIZE_THREAD__) -#if __has_feature(thread_sanitizer) /* used by clang */ -#define __SANITIZE_THREAD__ 1 /* GCC uses this; fake it in clang */ -#endif -#endif -#if defined(__SANITIZE_THREAD__) -NSYNC_C_START_ -void AnnotateIgnoreWritesBegin(const char *file, int line); -void AnnotateIgnoreWritesEnd(const char *file, int line); -void AnnotateIgnoreReadsBegin(const char *file, int line); -void AnnotateIgnoreReadsEnd(const char *file, int line); -NSYNC_C_END_ -#define IGNORE_RACES_START() \ - do { \ - AnnotateIgnoreReadsBegin(__FILE__, __LINE__); \ - AnnotateIgnoreWritesBegin(__FILE__, __LINE__); \ - } while (0) -#define IGNORE_RACES_END() \ - do { \ - AnnotateIgnoreWritesEnd(__FILE__, __LINE__); \ - AnnotateIgnoreReadsEnd(__FILE__, __LINE__); \ - } while (0) -#else -#define IGNORE_RACES_START() -#define IGNORE_RACES_END() -#endif - -COSMOPOLITAN_C_END_ -#endif /* !(__ASSEMBLER__ + __LINKER__ + 0) */ -#endif /* COSMOPOLITAN_THIRD_PARTY_NSYNC_RACES_INTERNAL_H_ */ diff --git a/third_party/nsync/time.c b/third_party/nsync/time.c deleted file mode 100644 index d401c23cc..000000000 --- a/third_party/nsync/time.c +++ /dev/null @@ -1,54 +0,0 @@ -/*-*- mode:c;indent-tabs-mode:t;c-basic-offset:8;tab-width:8;coding:utf-8 -*-│ -│vi: set et ft=c ts=8 tw=8 fenc=utf-8 :vi│ -╞══════════════════════════════════════════════════════════════════════════════╡ -│ Copyright 2016 Google Inc. │ -│ │ -│ Licensed under the Apache License, Version 2.0 (the "License"); │ -│ you may not use this file except in compliance with the License. │ -│ You may obtain a copy of the License at │ -│ │ -│ http://www.apache.org/licenses/LICENSE-2.0 │ -│ │ -│ Unless required by applicable law or agreed to in writing, software │ -│ distributed under the License is distributed on an "AS IS" BASIS, │ -│ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. │ -│ See the License for the specific language governing permissions and │ -│ limitations under the License. │ -╚─────────────────────────────────────────────────────────────────────────────*/ -#include "libc/calls/struct/timespec.h" -#include "libc/str/str.h" -#include "libc/sysv/consts/clock.h" -#include "third_party/nsync/time.h" - -asm(".ident\t\"\\n\\n\ -*NSYNC (Apache 2.0)\\n\ -Copyright 2016 Google, Inc.\\n\ -https://github.com/google/nsync\""); -// clang-format off - -#define NSYNC_NS_IN_S_ (1000 * 1000 * 1000) - -/* Return the maximum t, assuming it's an integral - type, and the representation is not too strange. */ -#define MAX_INT_TYPE(t) (((t)~(t)0) > 1? /*is t unsigned?*/ \ - (t)~(t)0 : /*unsigned*/ \ - (t) ((((uintmax_t)1) << (sizeof (t) * CHAR_BIT - 1)) - 1)) /*signed*/ - -const nsync_time nsync_time_no_deadline = - NSYNC_TIME_STATIC_INIT (MAX_INT_TYPE (int64_t), NSYNC_NS_IN_S_ - 1); - -const nsync_time nsync_time_zero = NSYNC_TIME_STATIC_INIT (0, 0); - -nsync_time nsync_time_sleep (nsync_time delay) { - struct timespec ts; - struct timespec remain; - memset (&ts, 0, sizeof (ts)); - ts.tv_sec = NSYNC_TIME_SEC (delay); - ts.tv_nsec = NSYNC_TIME_NSEC (delay); - if (nanosleep (&ts, &remain) == 0) { - /* nanosleep() is not required to fill in "remain" - if it returns 0. */ - memset (&remain, 0, sizeof (remain)); - } - return (remain); -} diff --git a/third_party/nsync/time.h b/third_party/nsync/time.h deleted file mode 100644 index 2529d5d87..000000000 --- a/third_party/nsync/time.h +++ /dev/null @@ -1,53 +0,0 @@ -#ifndef NSYNC_TIME_H_ -#define NSYNC_TIME_H_ -#include "libc/calls/struct/timespec.h" -#if !(__ASSEMBLER__ + __LINKER__ + 0) -COSMOPOLITAN_C_START_ - -#define NSYNC_TIME_SEC(t) ((t).tv_sec) -#define NSYNC_TIME_NSEC(t) ((t).tv_nsec) -#define NSYNC_TIME_STATIC_INIT(t, ns) \ - { (t), (ns) } - -/* The type nsync_time represents the interval elapsed between two - moments in time. Often the first such moment is an address-space-wide - epoch, such as the Unix epoch, but clients should not rely on the - epoch in one address space being the same as that in another. - Intervals relative to the epoch are known as absolute times. */ -typedef struct timespec nsync_time; - -/* A deadline infinitely far in the future. */ -extern const nsync_time nsync_time_no_deadline; - -/* The zero delay, or an expired deadline. */ -extern const nsync_time nsync_time_zero; - -/* Return the current time since the epoch. */ -#define nsync_time_now() _timespec_real() - -/* Sleep for the specified delay. Returns the unslept time which may be - non-zero if the call was interrupted. */ -nsync_time nsync_time_sleep(nsync_time delay); - -/* Return a+b */ -#define nsync_time_add(a, b) _timespec_add(a, b) - -/* Return a-b */ -#define nsync_time_sub(a, b) _timespec_sub(a, b) - -/* Return +ve, 0, or -ve according to whether a>b, a==b, or a the waiter is waiting */ - struct nsync_semaphore_s_ *sem; /* *sem will be Ved when waiter is woken */ - uint32_t flags; /* see below */ -}; - -/* set if waiter is embedded in Mu/CV's internal structures */ -#define NSYNC_WAITER_FLAG_MUCV 0x1 - -COSMOPOLITAN_C_END_ -#endif /* !(__ASSEMBLER__ + __LINKER__ + 0) */ -#endif /* COSMOPOLITAN_LIBC_THREAD_WAIT_INTERNAL_H_ */ diff --git a/third_party/nsync/waiter.h b/third_party/nsync/waiter.h deleted file mode 100644 index 057591f13..000000000 --- a/third_party/nsync/waiter.h +++ /dev/null @@ -1,138 +0,0 @@ -#ifndef NSYNC_WAITER_H_ -#define NSYNC_WAITER_H_ -#include "third_party/nsync/time.h" -#if !(__ASSEMBLER__ + __LINKER__ + 0) -COSMOPOLITAN_C_START_ - -/* nsync_wait_n() allows the client to wait on multiple objects - (condition variables, nsync_notes, nsync_counters, etc.) until at - least one of them becomes ready, or a deadline expires. - - It can be thought of as rather like Unix's select() or poll(), except - the the objects being waited for are synchronization data structures, - rather than file descriptors. - - The client can construct new objects that can be waited for by - implementing three routines. - - Examples: - - To wait on two nsync_notes n0, n1, and a nsync_counter c0, with a - deadline of abs_deadline: - - // Form an array of struct nsync_waitable_s, identifying the - // objects and the corresponding descriptors. (Static - // initialization syntax is used for brevity.) - static struct nsync_waitable_s w[] = { - { &n0, &nsync_note_waitable_funcs }, - { &n1, &nsync_note_waitable_funcs }, - { &c0, &nsync_counter_waitable_funcs } - }; - static struct nsync_waitable_s *pw[] = { &w[0], &w[1], &w[2] }; - int n = sizeof (w) / sizeof (w[0]); - - // Wait. The mu, lock, and unlock arguments are NULL because no - // condition variables are invovled. - int i = nsync_wait_n (NULL, NULL, NULL, abs_deadline, n, pw); - if (i == n) { - // timeout - } else { - // w[i].v became ready. - } - - To wait on multiple condition variables, the mu/lock/unlock - parameters are used. Imagine cv0 and cv1 are signalled when - predicates pred0() (under lock mu0) and pred1() (under lock mu1) - become true respectively. Assume that mu0 is acquired before mu1. - - static void lock2 (void *v) { // lock two mutexes in order - nsync_mu **mu = (nsync_mu **) v; - nsync_mu_lock (mu[0]); - nsync_mu_lock (mu[1]); - } - static void unlock2 (void *v) { // unlock two mutexes. - nsync_mu **mu = (nsync_mu **) v; - nsync_mu_unlock (mu[1]); - nsync_mu_unlock (mu[0]); - } - - // Describe the condition variables and the locks. - static struct nsync_waitable_s w[] = { - { &cv0, &nsync_cv_waitable_funcs }, - { &cv1, &nsync_cv_waitable_funcs } - }; - static struct nsync_waitable_s *pw[] = { &w[0], &w[1] }; - nsync_mu *lock_list[] = { &mu0, &mu1 }; - int n = sizeof (w) / sizeof (w[0]); - - lock2 (list_list); - while (!pred0 () && !pred1 ()) { - // Wait for one of the condition variables to be signalled, - // with no timeout. - nsync_wait_n (lock_list, &lock2, &unlock2, - nsync_time_no_deadline, n, pw); - } - if (pred0 ()) { ... } - if (pred1 ()) { ... } - unlock2 (list_list); - - */ - -/* forward declaration of struct that contains type dependent wait - operations */ -struct nsync_waitable_funcs_s; - -/* Clients wait on objects by forming an array of struct - nsync_waitable_s. Each each element points to one object and its - type-dependent functions. */ -struct nsync_waitable_s { - /* pointer to object */ - void *v; - /* pointer to type-dependent functions. Use - &nsync_note_waitable_funcs for an nsync_note, - &nsync_counternote_waitable_funcs for an nsync_counter, - &nsync_cv_waitable_funcs for an nsync_cv. */ - const struct nsync_waitable_funcs_s *funcs; -}; - -/* Wait until at least one of *waitable[0,..,count-1] is has been - notified, or abs_deadline is reached. Return the index of the - notified element of waitable[], or count if no such element exists. - If mu!=NULL, (*unlock)(mu) is called after the thread is queued on - the various waiters, and (*lock)(mu) is called before return; - mu/lock/unlock are used to acquire and release the relevant locks - whan waiting on condition variables. */ -int nsync_wait_n(void *mu, void (*lock)(void *), void (*unlock)(void *), - nsync_time abs_deadline, int count, - struct nsync_waitable_s *waitable[]); - -/* A "struct nsync_waitable_s" implementation must implement these - functions. Clients should ignore the internals. */ -struct nsync_waiter_s; -struct nsync_waitable_funcs_s { - - /* Return the time when *v will be ready (max time if unknown), or 0 - if it is already ready. The parameter nw may be passed as NULL, in - which case the result should indicate whether the thread would - block if it were to wait on *v. All calls with the same *v must - report the same result until the object becomes ready, from which - point calls must report 0. */ - nsync_time (*ready_time)(void *v, struct nsync_waiter_s *nw); - - /* If *v is ready, return zero; otherwise enqueue *nw on *v and return - non-zero. */ - int (*enqueue)(void *v, struct nsync_waiter_s *nw); - - /* If nw has been previously dequeued, return zero; otherwise dequeue - *nw from *v and return non-zero. */ - int (*dequeue)(void *v, struct nsync_waiter_s *nw); -}; - -/* The "struct nsync_waitable_s" for nsync_note, nsync_counter, and nsync_cv. */ -extern const struct nsync_waitable_funcs_s nsync_note_waitable_funcs; -extern const struct nsync_waitable_funcs_s nsync_counter_waitable_funcs; -extern const struct nsync_waitable_funcs_s nsync_cv_waitable_funcs; - -COSMOPOLITAN_C_END_ -#endif /* !(__ASSEMBLER__ + __LINKER__ + 0) */ -#endif /* NSYNC_WAITER_H_ */ diff --git a/third_party/nsync/waiter_per_thread.c b/third_party/nsync/waiter_per_thread.c deleted file mode 100644 index 3ac2772f1..000000000 --- a/third_party/nsync/waiter_per_thread.c +++ /dev/null @@ -1,56 +0,0 @@ -/*-*- mode:c;indent-tabs-mode:t;c-basic-offset:8;tab-width:8;coding:utf-8 -*-│ -│vi: set et ft=c ts=8 tw=8 fenc=utf-8 :vi│ -╞══════════════════════════════════════════════════════════════════════════════╡ -│ Copyright 2016 Google Inc. │ -│ │ -│ Licensed under the Apache License, Version 2.0 (the "License"); │ -│ you may not use this file except in compliance with the License. │ -│ You may obtain a copy of the License at │ -│ │ -│ http://www.apache.org/licenses/LICENSE-2.0 │ -│ │ -│ Unless required by applicable law or agreed to in writing, software │ -│ distributed under the License is distributed on an "AS IS" BASIS, │ -│ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. │ -│ See the License for the specific language governing permissions and │ -│ limitations under the License. │ -╚─────────────────────────────────────────────────────────────────────────────*/ -#include "third_party/nsync/atomic.h" -#include "third_party/nsync/atomic.internal.h" -#include "third_party/nsync/common.internal.h" -#include "libc/thread/thread.h" - -asm(".ident\t\"\\n\\n\ -*NSYNC (Apache 2.0)\\n\ -Copyright 2016 Google, Inc.\\n\ -https://github.com/google/nsync\""); -// clang-format off - -static pthread_key_t waiter_key; -static nsync_atomic_uint32_ pt_once; - -static void do_once (nsync_atomic_uint32_ *ponce, void (*dest) (void *)) { - uint32_t o = ATM_LOAD_ACQ (ponce); - if (o != 2) { - while (o == 0 && !ATM_CAS_ACQ (ponce, 0, 1)) { - o = ATM_LOAD (ponce); - } - if (o == 0) { - pthread_key_create (&waiter_key, dest); - ATM_STORE_REL (ponce, 2); - } - while (ATM_LOAD_ACQ (ponce) != 2) { - nsync_yield_ (); - } - } -} - -void *nsync_per_thread_waiter_ (void (*dest) (void *)) { - do_once (&pt_once, dest); - return (pthread_getspecific (waiter_key)); -} - -void nsync_set_per_thread_waiter_ (void *v, void (*dest) (void *)) { - do_once (&pt_once, dest); - pthread_setspecific (waiter_key, v); -} diff --git a/third_party/nsync/yield.c b/third_party/nsync/yield.c deleted file mode 100644 index b6183a85c..000000000 --- a/third_party/nsync/yield.c +++ /dev/null @@ -1,27 +0,0 @@ -/*-*- mode:c;indent-tabs-mode:t;c-basic-offset:8;tab-width:8;coding:utf-8 -*-│ -│vi: set et ft=c ts=8 tw=8 fenc=utf-8 :vi│ -╞══════════════════════════════════════════════════════════════════════════════╡ -│ Copyright 2022 Justine Alexandra Roberts Tunney │ -│ │ -│ Permission to use, copy, modify, and/or distribute this software for │ -│ any purpose with or without fee is hereby granted, provided that the │ -│ above copyright notice and this permission notice appear in all copies. │ -│ │ -│ THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL │ -│ WARRANTIES WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED │ -│ WARRANTIES OF MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE │ -│ AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL │ -│ DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR │ -│ PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER │ -│ TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR │ -│ PERFORMANCE OF THIS SOFTWARE. │ -╚─────────────────────────────────────────────────────────────────────────────*/ -#include "libc/calls/calls.h" -#include "libc/calls/strace.internal.h" -#include "third_party/nsync/common.internal.h" -// clang-format off - -void nsync_yield_ (void) { - sched_yield (); - STRACE ("nsync_yield_()"); -} diff --git a/third_party/python/Objects/fileobject.c b/third_party/python/Objects/fileobject.c index 4f591d0f9..5f3186f27 100644 --- a/third_party/python/Objects/fileobject.c +++ b/third_party/python/Objects/fileobject.c @@ -7,7 +7,7 @@ #define PY_SSIZE_T_CLEAN #include "libc/calls/calls.h" #include "libc/errno.h" -#include "libc/stdio/lock.internal.h" +#include "libc/stdio/lock.h" #include "third_party/python/Include/abstract.h" #include "third_party/python/Include/boolobject.h" #include "third_party/python/Include/bytesobject.h" diff --git a/third_party/sed/compile.c b/third_party/sed/compile.c index fe9fa8f11..aaa7a950a 100644 --- a/third_party/sed/compile.c +++ b/third_party/sed/compile.c @@ -63,7 +63,7 @@ #include "third_party/regex/regex.h" #include "libc/calls/calls.h" #include "libc/fmt/fmt.h" -#include "libc/stdio/lock.internal.h" +#include "libc/stdio/lock.h" #include "libc/stdio/stdio.h" #include "libc/stdio/temp.h" #include "libc/mem/alg.h" diff --git a/third_party/sed/main.c b/third_party/sed/main.c index e525ebb5f..cf0b97318 100644 --- a/third_party/sed/main.c +++ b/third_party/sed/main.c @@ -91,7 +91,7 @@ #define _WITH_GETLINE #include "libc/calls/calls.h" #include "libc/fmt/fmt.h" -#include "libc/stdio/lock.internal.h" +#include "libc/stdio/lock.h" #include "libc/stdio/stdio.h" #include "libc/stdio/temp.h" #include "libc/mem/alg.h" diff --git a/third_party/sed/misc.c b/third_party/sed/misc.c index 52b676c2a..1548bfdc9 100644 --- a/third_party/sed/misc.c +++ b/third_party/sed/misc.c @@ -48,7 +48,7 @@ #include "third_party/regex/regex.h" #include "libc/calls/calls.h" #include "libc/fmt/fmt.h" -#include "libc/stdio/lock.internal.h" +#include "libc/stdio/lock.h" #include "libc/stdio/stdio.h" #include "libc/stdio/temp.h" #include "libc/mem/alg.h" diff --git a/third_party/sed/process.c b/third_party/sed/process.c index 7b1d3d750..e5199a72c 100644 --- a/third_party/sed/process.c +++ b/third_party/sed/process.c @@ -74,7 +74,7 @@ #include "third_party/regex/regex.h" #include "libc/calls/calls.h" #include "libc/fmt/fmt.h" -#include "libc/stdio/lock.internal.h" +#include "libc/stdio/lock.h" #include "libc/stdio/stdio.h" #include "libc/stdio/temp.h" #include "libc/mem/alg.h" diff --git a/third_party/sqlite3/main.c b/third_party/sqlite3/main.c index 02e7fe6a8..31673e193 100644 --- a/third_party/sqlite3/main.c +++ b/third_party/sqlite3/main.c @@ -3105,10 +3105,6 @@ static int openDatabase( char *zErrMsg = 0; /* Error message from sqlite3ParseUri() */ int i; /* Loop counter */ - // TODO(jart): Fix SQLite. - extern bool __force_sqlite_to_work_until_we_can_fix_it; - __force_sqlite_to_work_until_we_can_fix_it = true; - #ifdef SQLITE_ENABLE_API_ARMOR if( ppDb==0 ) return SQLITE_MISUSE_BKPT; #endif diff --git a/third_party/third_party.mk b/third_party/third_party.mk index e42a207c0..c7f65bec0 100644 --- a/third_party/third_party.mk +++ b/third_party/third_party.mk @@ -20,13 +20,12 @@ o/$(MODE)/third_party: \ o/$(MODE)/third_party/maxmind \ o/$(MODE)/third_party/mbedtls \ o/$(MODE)/third_party/musl \ - o/$(MODE)/third_party/nsync \ o/$(MODE)/third_party/python \ o/$(MODE)/third_party/quickjs \ o/$(MODE)/third_party/regex \ - o/$(MODE)/third_party/sed \ o/$(MODE)/third_party/smallz4 \ o/$(MODE)/third_party/sqlite3 \ + o/$(MODE)/third_party/sed \ o/$(MODE)/third_party/stb \ o/$(MODE)/third_party/tidy \ o/$(MODE)/third_party/unzip \ diff --git a/third_party/zip/zipfile.c b/third_party/zip/zipfile.c index 47c0b4231..9a85e5b1d 100644 --- a/third_party/zip/zipfile.c +++ b/third_party/zip/zipfile.c @@ -17,7 +17,7 @@ #include "third_party/zip/zip.h" #include "third_party/zip/revision.h" #ifdef UNICODE_SUPPORT -#include "libc/stdio/lock.internal.h" +#include "libc/stdio/lock.h" #include "third_party/zip/crc32.h" #endif diff --git a/tool/build/lib/pty.c b/tool/build/lib/pty.c index 32d63cccc..264bb0ecc 100644 --- a/tool/build/lib/pty.c +++ b/tool/build/lib/pty.c @@ -641,18 +641,15 @@ static void PtyDeleteLines(struct Pty *pty) { } static void PtyReportDeviceStatus(struct Pty *pty) { - static const char report[] = "\e[0n"; - PtyWriteInput(pty, report, strlen(report)); + PtyWriteInput(pty, "\e[0n", 4); } static void PtyReportPreferredVtType(struct Pty *pty) { - static const char report[] = "\e[?1;0c"; - PtyWriteInput(pty, report, strlen(report)); + PtyWriteInput(pty, "\e[?1;0c", 4); } static void PtyReportPreferredVtIdentity(struct Pty *pty) { - static const char report[] = "\e/Z"; - PtyWriteInput(pty, report, strlen(report)); + PtyWriteInput(pty, "\e/Z", 4); } static void PtyBell(struct Pty *pty) { diff --git a/tool/build/mkdeps.c b/tool/build/mkdeps.c index b55bc74b2..e2876b925 100644 --- a/tool/build/mkdeps.c +++ b/tool/build/mkdeps.c @@ -24,7 +24,9 @@ #include "libc/fmt/fmt.h" #include "libc/intrin/bits.h" #include "libc/intrin/kprintf.h" +#include "libc/intrin/pthread.h" #include "libc/intrin/safemacros.internal.h" +#include "libc/intrin/wait0.internal.h" #include "libc/log/check.h" #include "libc/log/log.h" #include "libc/macros.internal.h" @@ -35,6 +37,7 @@ #include "libc/mem/bisectcarleft.internal.h" #include "libc/mem/mem.h" #include "libc/nexgen32e/crc32.h" +#include "libc/nexgen32e/threaded.h" #include "libc/runtime/ezmap.internal.h" #include "libc/runtime/gc.internal.h" #include "libc/runtime/runtime.h" @@ -49,9 +52,6 @@ #include "libc/sysv/consts/o.h" #include "libc/sysv/consts/prot.h" #include "libc/thread/spawn.h" -#include "libc/thread/thread.h" -#include "libc/thread/tls.h" -#include "libc/thread/wait0.internal.h" #include "libc/time/time.h" #include "libc/x/x.h" #include "third_party/getopt/getopt.h" diff --git a/tool/build/pstrace.c b/tool/build/pstrace.c index 3ba54a7ac..78bb2c572 100644 --- a/tool/build/pstrace.c +++ b/tool/build/pstrace.c @@ -16,12 +16,12 @@ │ TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR │ │ PERFORMANCE OF THIS SOFTWARE. │ ╚─────────────────────────────────────────────────────────────────────────────*/ +#include "libc/intrin/bits.h" #include "libc/calls/calls.h" #include "libc/calls/struct/sigaction.h" #include "libc/calls/struct/sigset.h" #include "libc/errno.h" #include "libc/fmt/conv.h" -#include "libc/intrin/bits.h" #include "libc/log/check.h" #include "libc/log/log.h" #include "libc/mem/mem.h" @@ -115,7 +115,7 @@ struct Trace { EK_KILLED, // ret is signal code } kind; unsigned char arity; - unsigned char syscall_; + unsigned char syscall; bool is_interrupted; int us; int elap; @@ -675,7 +675,7 @@ static void Parse(struct Trace *t, const char *line, long lineno) { return; } else if (isalpha(*p) && (q = strchr(p, '('))) { t->events.p[event].kind = EK_CALL; - CHECK_NE(-1, (t->events.p[event].syscall_ = GetSyscall(p, q - p)), DEBUG); + CHECK_NE(-1, (t->events.p[event].syscall = GetSyscall(p, q - p)), DEBUG); p = q + 1; } } @@ -825,8 +825,8 @@ static void PrintEvent(FILE *f, struct Trace *t, long ev) { fprintf(f, "%d", t->events.p[ev].ret); break; case EK_CALL: - CHECK_LT(t->events.p[ev].syscall_, ARRAYLEN(kSyscalls)); - fprintf(f, "(b%`'s,%ld,", kSyscalls[t->events.p[ev].syscall_].name, + CHECK_LT(t->events.p[ev].syscall, ARRAYLEN(kSyscalls)); + fprintf(f, "(b%`'s,%ld,", kSyscalls[t->events.p[ev].syscall].name, t->events.p[ev].ret); fprintf(f, "%c", t->events.p[ev].arity && t->events.p[ev].arg[0].name != -1 ? '{' diff --git a/tool/emacs/cosmo-stuff.el b/tool/emacs/cosmo-stuff.el index d7241f477..a10d4b269 100644 --- a/tool/emacs/cosmo-stuff.el +++ b/tool/emacs/cosmo-stuff.el @@ -637,7 +637,7 @@ (let ((default-directory root)) (save-buffer) (cond ((memq major-mode '(c-mode c++-mode asm-mode fortran-mode)) - (let* ((mode (cosmo--make-mode arg "fastbuild")) + (let* ((mode (cosmo--make-mode arg "")) (compile-command (cosmo--compile-command this root 'test mode "" "" ".ok"))) (compile compile-command))) ('t @@ -653,7 +653,7 @@ (let ((default-directory root)) (save-buffer) (cond ((memq major-mode '(c-mode c++-mode asm-mode fortran-mode)) - (let* ((mode (cosmo--make-mode arg "fastbuild")) + (let* ((mode (cosmo--make-mode arg "")) (compile-command (cosmo--compile-command this root 'run-win10 mode "" "" ""))) (compile compile-command))) ('t diff --git a/tool/net/help.txt b/tool/net/help.txt index 3d1971059..4a4f32491 100644 --- a/tool/net/help.txt +++ b/tool/net/help.txt @@ -792,7 +792,7 @@ FUNCTIONS The following options may be used: - useoutput: (bool=false) encodes the result directly to the - output buffer and returns `true` value. This option is + output buffer and returns `nil` value. This option is ignored if used outside of request handling code. - sorted: (bool=true) Lua uses hash tables so the order of @@ -801,13 +801,13 @@ FUNCTIONS don't care about ordering then setting `sorted=false` should yield a performance boost in serialization. - - pretty: (bool=false) Setting this option to `true` will + - pretty: (bool=false) Setting this option to true will cause tables with more than one entry to be formatted across multiple lines for readability. - indent: (str=" ") This option controls the indentation of pretty formatting. This field is ignored if `pretty` isn't - `true`. + true. - maxdepth: (int=64) This option controls the maximum amount of recursion the serializer is allowed to perform. The max @@ -863,7 +863,7 @@ FUNCTIONS The following options may be used: - useoutput: (bool=false) encodes the result directly to the - output buffer and returns `true` value. This option is + output buffer and returns `nil` value. This option is ignored if used outside of request handling code. - sorted: (bool=true) Lua uses hash tables so the order of @@ -872,13 +872,13 @@ FUNCTIONS don't care about ordering then setting `sorted=false` should yield a performance boost in serialization. - - pretty: (bool=false) Setting this option to `true` will + - pretty: (bool=false) Setting this option to true will cause tables with more than one entry to be formatted across multiple lines for readability. - indent: (str=" ") This option controls the indentation of pretty formatting. This field is ignored if `pretty` isn't - `true`. + true. - maxdepth: (int=64) This option controls the maximum amount of recursion the serializer is allowed to perform. The max diff --git a/tool/net/redbean.c b/tool/net/redbean.c index 15775acff..890ec1321 100644 --- a/tool/net/redbean.c +++ b/tool/net/redbean.c @@ -16,7 +16,6 @@ │ TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR │ │ PERFORMANCE OF THIS SOFTWARE. │ ╚─────────────────────────────────────────────────────────────────────────────*/ -#include "libc/atomic.h" #include "libc/calls/calls.h" #include "libc/calls/ioctl.h" #include "libc/calls/pledge.h" @@ -37,6 +36,7 @@ #include "libc/intrin/atomic.h" #include "libc/intrin/likely.h" #include "libc/intrin/nomultics.internal.h" +#include "libc/intrin/pthread.h" #include "libc/intrin/safemacros.internal.h" #include "libc/log/check.h" #include "libc/log/log.h" @@ -48,6 +48,7 @@ #include "libc/nexgen32e/crc32.h" #include "libc/nexgen32e/nt2sysv.h" #include "libc/nexgen32e/rdtsc.h" +#include "libc/nexgen32e/threaded.h" #include "libc/nexgen32e/x86feature.h" #include "libc/nt/enum/fileflagandattributes.h" #include "libc/nt/thread.h" @@ -90,8 +91,6 @@ #include "libc/sysv/consts/w.h" #include "libc/sysv/errfuns.h" #include "libc/thread/spawn.h" -#include "libc/thread/thread.h" -#include "libc/thread/tls.h" #include "libc/x/x.h" #include "libc/zip.h" #include "net/http/escape.h" @@ -419,8 +418,8 @@ static bool hasonprocessdestroy; static bool loggednetworkorigin; static bool ishandlingconnection; static bool hasonclientconnection; -static atomic_bool terminatemonitor; static bool evadedragnetsurveillance; +static _Atomic(bool) terminatemonitor; static int zfd; static int gmtoff; diff --git a/tool/plinko/plinko.c b/tool/plinko/plinko.c index 6a75105c4..7a6120a16 100644 --- a/tool/plinko/plinko.c +++ b/tool/plinko/plinko.c @@ -16,7 +16,7 @@ │ TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR │ │ PERFORMANCE OF THIS SOFTWARE. │ ╚─────────────────────────────────────────────────────────────────────────────*/ -#include "libc/thread/tls.h" +#include "libc/nexgen32e/threaded.h" #include "tool/plinko/lib/plinko.h" STATIC_YOINK("__zipos_get"); diff --git a/tool/scripts/check-includes.py b/tool/scripts/check-includes.py new file mode 100755 index 000000000..eed104417 --- /dev/null +++ b/tool/scripts/check-includes.py @@ -0,0 +1,20 @@ +#!/usr/bin/env python.com + +import os +import sys + +def CheckFile(path): + if path.endswith(('.png', '.ico')): + return + sys.stderr.write('%s\n' % (path)) + with open(path) as f: + data = f.read() + assert '#include' not in data[65530:], "late include in %s" % (path) + +for arg in sys.argv[1:]: + if os.path.isdir(arg): + for dirpath, dirs, files in os.walk(arg): + for filepath in files: + CheckFile(os.path.join(dirpath, filepath)) + else: + CheckFile(arg)