mirrors/cosmopolitan
1
0
Fork 0
mirror of https://github.com/jart/cosmopolitan.git synced 2025-07-27 04:50:28 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

Improve threading support further

Browse source
This commit is contained in:
Justine Tunney 2022-05-17 04:14:28 -07:00
parent 8bfb70ca3f
commit ce71677156
61 changed files with 882 additions and 747 deletions
Download patch file Download diff file Expand all files Collapse all files

6
libc/calls/close.c
View file

@ -20,6 +20,8 @@
#include "libc/calls/calls.h"
#include "libc/calls/internal.h"
#include "libc/calls/strace.internal.h"
#include "libc/intrin/kprintf.h"
#include "libc/intrin/spinlock.h"
#include "libc/macros.internal.h"
#include "libc/sock/internal.h"
#include "libc/sysv/errfuns.h"
@ -46,6 +48,7 @@
*/
int close(int fd) {
int rc;
_spinlock(&__fds_lock);
if (fd == -1) {
rc = 0;
} else if (fd < 0) {
@ -74,9 +77,10 @@ int close(int fd) {
}
}
if (!__vforked) {
__releasefd(fd);
__releasefd_unlocked(fd);
}
}
_spunlock(&__fds_lock);
STRACE("%s(%d) → %d% m", "close", fd, rc);
return rc;
}

28
libc/calls/dup-nt.c
View file

@ -20,6 +20,7 @@
#include "libc/bits/weaken.h"
#include "libc/calls/calls.h"
#include "libc/calls/internal.h"
#include "libc/intrin/spinlock.h"
#include "libc/mem/mem.h"
#include "libc/nt/files.h"
#include "libc/nt/runtime.h"
@ -32,25 +33,37 @@
* Implements dup(), dup2(), dup3(), and F_DUPFD for Windows.
*/
textwindows int sys_dup_nt(int oldfd, int newfd, int flags, int start) {
int64_t proc, handle;
int64_t rc, proc, handle;
// validate the api usage
if (oldfd < 0) return einval();
if (flags & ~O_CLOEXEC) return einval();
_spinlock(&__fds_lock);
if (oldfd >= g_fds.n ||
(g_fds.p[oldfd].kind != kFdFile && g_fds.p[oldfd].kind != kFdSocket &&
g_fds.p[oldfd].kind != kFdConsole)) {
_spunlock(&__fds_lock);
return ebadf();
}
// allocate a new file descriptor
if (newfd == -1) {
if ((newfd = __reservefd(start)) == -1) {
if ((newfd = __reservefd_unlocked(start)) == -1) {
_spunlock(&__fds_lock);
return -1;
}
} else {
if (__ensurefds(newfd) == -1) return -1;
if (g_fds.p[newfd].kind) close(newfd);
if (__ensurefds_unlocked(newfd) == -1) {
_spunlock(&__fds_lock);
return -1;
}
if (g_fds.p[newfd].kind) {
_spunlock(&__fds_lock);
close(newfd);
_spinlock(&__fds_lock);
}
g_fds.p[newfd].kind = kFdReserved;
}
@ -80,9 +93,12 @@ textwindows int sys_dup_nt(int oldfd, int newfd, int flags, int start) {
if (g_fds.p[oldfd].worker) {
g_fds.p[newfd].worker = weaken(RefNtStdinWorker)(g_fds.p[oldfd].worker);
}
return newfd;
rc = newfd;
} else {
__releasefd(newfd);
return __winerr();
rc = __winerr();
}
_spunlock(&__fds_lock);
return rc;
}

2
libc/calls/g_sighandrvas.c
View file

@ -18,6 +18,6 @@
╚─────────────────────────────────────────────────────────────────────────────*/
#include "libc/calls/internal.h"
_Alignas(64) char __sig_lock;
_Alignas(64) int __sig_lock;
unsigned __sighandrvas[NSIG];
unsigned __sighandflags[NSIG];

7
libc/calls/internal.h
View file

@ -76,8 +76,8 @@ struct Fds {
extern const struct Fd kEmptyFd;
hidden extern int __vforked;
hidden extern char __fds_lock;
hidden extern char __sig_lock;
hidden extern int __fds_lock;
hidden extern int __sig_lock;
hidden extern bool __time_critical;
hidden extern unsigned __sighandrvas[NSIG];
hidden extern unsigned __sighandflags[NSIG];
@ -85,8 +85,11 @@ hidden extern struct Fds g_fds;
hidden extern const struct NtSecurityAttributes kNtIsInheritable;
int __reservefd(int) hidden;
int __reservefd_unlocked(int) hidden;
void __releasefd(int) hidden;
void __releasefd_unlocked(int) hidden;
int __ensurefds(int) hidden;
int __ensurefds_unlocked(int) hidden;
int64_t __getfdhandleactual(int) hidden;
void __printfds(void) hidden;

7
libc/calls/interrupts-nt.c
View file

@ -23,11 +23,16 @@
#include "libc/calls/strace.internal.h"
#include "libc/calls/struct/sigaction.h"
#include "libc/dce.h"
#include "libc/intrin/spinlock.h"
textwindows bool _check_interrupts(bool restartable, struct Fd *fd) {
bool res;
if (__time_critical) return false;
if (_trylock(&__fds_lock)) return false;
if (weaken(_check_sigalrm)) weaken(_check_sigalrm)();
if (weaken(_check_sigchld)) weaken(_check_sigchld)();
if (fd && weaken(_check_sigwinch)) weaken(_check_sigwinch)(fd);
return weaken(__sig_check) && weaken(__sig_check)(restartable);
res = weaken(__sig_check) && weaken(__sig_check)(restartable);
_spunlock(&__fds_lock);
return res;
}

20
libc/calls/open-nt.c
View file

@ -20,6 +20,7 @@
#include "libc/calls/internal.h"
#include "libc/calls/ntmagicpaths.internal.h"
#include "libc/calls/strace.internal.h"
#include "libc/intrin/spinlock.h"
#include "libc/nt/createfile.h"
#include "libc/nt/enum/accessmask.h"
#include "libc/nt/enum/creationdisposition.h"
@ -87,14 +88,17 @@ textwindows ssize_t sys_open_nt(int dirfd, const char *file, uint32_t flags,
int32_t mode) {
int fd;
ssize_t rc;
if ((fd = __reservefd(-1)) == -1) return -1;
if ((flags & O_ACCMODE) == O_RDWR && !strcmp(file, kNtMagicPaths.devtty)) {
rc = sys_open_nt_console(dirfd, &kNtMagicPaths, flags, mode, fd);
} else {
rc = sys_open_nt_file(dirfd, file, flags, mode, fd);
}
if (rc == -1) {
__releasefd(fd);
_spinlock(&__fds_lock);
if ((rc = fd = __reservefd_unlocked(-1)) != -1) {
if ((flags & O_ACCMODE) == O_RDWR && !strcmp(file, kNtMagicPaths.devtty)) {
rc = sys_open_nt_console(dirfd, &kNtMagicPaths, flags, mode, fd);
} else {
rc = sys_open_nt_file(dirfd, file, flags, mode, fd);
}
if (rc == -1) {
__releasefd_unlocked(fd);
}
_spunlock(&__fds_lock);
}
return rc;
}

18
libc/calls/pipe-nt.c
View file

@ -17,6 +17,7 @@
│ PERFORMANCE OF THIS SOFTWARE. │
╚─────────────────────────────────────────────────────────────────────────────*/
#include "libc/calls/internal.h"
#include "libc/intrin/spinlock.h"
#include "libc/nt/createfile.h"
#include "libc/nt/enum/accessmask.h"
#include "libc/nt/enum/creationdisposition.h"
@ -33,9 +34,14 @@ textwindows int sys_pipe_nt(int pipefd[2], unsigned flags) {
int reader, writer;
char16_t pipename[64];
CreatePipeName(pipename);
if ((reader = __reservefd(-1)) == -1) return -1;
if ((writer = __reservefd(-1)) == -1) {
__releasefd(reader);
_spinlock(&__fds_lock);
if ((reader = __reservefd_unlocked(-1)) == -1) {
_spunlock(&__fds_lock);
return -1;
}
if ((writer = __reservefd_unlocked(-1)) == -1) {
__releasefd_unlocked(reader);
_spunlock(&__fds_lock);
return -1;
}
if (~flags & O_DIRECT) {
@ -58,12 +64,14 @@ textwindows int sys_pipe_nt(int pipefd[2], unsigned flags) {
g_fds.p[writer].handle = hout;
pipefd[0] = reader;
pipefd[1] = writer;
_spunlock(&__fds_lock);
return 0;
} else {
CloseHandle(hin);
}
}
__releasefd(writer);
__releasefd(reader);
__releasefd_unlocked(writer);
__releasefd_unlocked(reader);
_spunlock(&__fds_lock);
return -1;
}

2
libc/calls/poll-nt.c
View file

@ -44,7 +44,7 @@
#include "libc/sysv/consts/sig.h"
#include "libc/sysv/errfuns.h"
_Alignas(64) static char poll_lock;
_Alignas(64) static int poll_lock;
/**
* Polls on the New Technology.

55
libc/calls/reservefd.c
View file

@ -32,10 +32,9 @@
/**
* Grows file descriptor array memory if needed.
*/
int __ensurefds(int fd) {
int __ensurefds_unlocked(int fd) {
size_t n1, n2;
struct Fd *p1, *p2;
_spinlock(&__fds_lock);
n1 = g_fds.n;
if (fd >= n1) {
STRACE("__ensurefds(%d) extending", fd);
@ -48,7 +47,7 @@ int __ensurefds(int fd) {
g_fds.p = p2;
g_fds.n = n2;
if (p1 != g_fds.__init_p) {
weaken(free)(p1);
__cxa_atexit(free, p1, 0);
}
} else {
fd = enomem();
@ -57,32 +56,44 @@ int __ensurefds(int fd) {
fd = emfile();
}
}
return fd;
}
/**
* Grows file descriptor array memory if needed.
*/
int __ensurefds(int fd) {
_spinlock(&__fds_lock);
fd = __ensurefds_unlocked(fd);
_spunlock(&__fds_lock);
return fd;
}
/**
* Finds open file descriptor slot.
*/
int __reservefd_unlocked(int start) {
int fd;
for (fd = g_fds.f; fd < g_fds.n; ++fd) {
if (!g_fds.p[fd].kind) {
break;
}
}
fd = __ensurefds_unlocked(fd);
bzero(g_fds.p + fd, sizeof(*g_fds.p));
g_fds.p[fd].kind = kFdReserved;
return fd;
}
/**
* Finds open file descriptor slot.
*/
int __reservefd(int start) {
int fd;
for (;;) {
_spinlock(&__fds_lock);
fd = start < 0 ? g_fds.f : start;
while (fd < g_fds.n && g_fds.p[fd].kind) ++fd;
if (fd < g_fds.n) {
g_fds.f = fd + 1;
bzero(g_fds.p + fd, sizeof(*g_fds.p));
g_fds.p[fd].kind = kFdReserved;
_spunlock(&__fds_lock);
return fd;
} else {
_spunlock(&__fds_lock);
if (__ensurefds(fd) == -1) {
return -1;
}
}
}
_spinlock(&__fds_lock);
fd = __reservefd_unlocked(start);
_spunlock(&__fds_lock);
return fd;
}
/**
@ -91,9 +102,12 @@ int __reservefd(int start) {
static void FreeFds(void) {
int i;
NTTRACE("FreeFds()");
_spinlock(&__fds_lock);
for (i = 3; i < g_fds.n; ++i) {
if (g_fds.p[i].kind) {
_spunlock(&__fds_lock);
close(i);
_spinlock(&__fds_lock);
}
}
if (g_fds.p != g_fds.__init_p) {
@ -102,6 +116,7 @@ static void FreeFds(void) {
g_fds.p = g_fds.__init_p;
g_fds.n = ARRAYLEN(g_fds.__init_p);
}
_spunlock(&__fds_lock);
}
static textstartup void FreeFdsInit(void) {

4
libc/calls/sigaction.c
View file

@ -223,7 +223,6 @@ static int __sigaction(int sig, const struct sigaction *act,
rc = 0;
}
if (rc != -1 && !__vforked) {
_spinlock(&__sig_lock);
if (oldact) {
oldrva = __sighandrvas[sig];
oldact->sa_sigaction = (sigaction_f)(
@ -233,7 +232,6 @@ static int __sigaction(int sig, const struct sigaction *act,
__sighandrvas[sig] = rva;
__sighandflags[sig] = act->sa_flags;
}
_spunlock(&__sig_lock);
}
return rc;
}
@ -447,7 +445,9 @@ int sigaction(int sig, const struct sigaction *act, struct sigaction *oldact) {
if (sig == SIGKILL || sig == SIGSTOP) {
rc = einval();
} else {
_spinlock(&__sig_lock);
rc = __sigaction(sig, act, oldact);
_spunlock(&__sig_lock);
}
STRACE("sigaction(%G, %s, [%s]) → %d% m", sig,
DescribeSigaction(buf[0], sizeof(buf[0]), 0, act),

2
libc/calls/sigchld-nt.c
View file

@ -53,7 +53,7 @@ void _check_sigchld(void) {
if (__sighandflags[SIGCHLD] & SA_NOCLDWAIT) {
STRACE("SIGCHILD SA_NOCLDWAIT fd=%d handle=%ld", pids[i], handles[i]);
CloseHandle(handles[i]);
__releasefd(pids[i]);
__releasefd_unlocked(pids[i]);
}
g_fds.p[pids[i]].zombie = true;
__sig_add(SIGCHLD, CLD_EXITED);

12
libc/calls/wait4-nt.c
View file

@ -25,6 +25,7 @@
#include "libc/calls/struct/rusage.h"
#include "libc/fmt/conv.h"
#include "libc/intrin/kprintf.h"
#include "libc/intrin/spinlock.h"
#include "libc/macros.internal.h"
#include "libc/nt/accounting.h"
#include "libc/nt/enum/accessmask.h"
@ -57,6 +58,7 @@ static textwindows int sys_wait4_nt_impl(int pid, int *opt_out_wstatus,
struct NtProcessMemoryCountersEx memcount;
struct NtFileTime createfiletime, exitfiletime, kernelfiletime, userfiletime;
if (_check_interrupts(true, g_fds.p)) return eintr();
_spinlock(&__fds_lock);
if (pid != -1 && pid != 0) {
if (pid < 0) {
/* XXX: this is sloppy */
@ -67,15 +69,17 @@ static textwindows int sys_wait4_nt_impl(int pid, int *opt_out_wstatus,
if (!__isfdopen(pid) &&
(handle = OpenProcess(kNtSynchronize | kNtProcessQueryInformation,
true, pid))) {
if ((pid = __reservefd(-1)) != -1) {
if ((pid = __reservefd_unlocked(-1)) != -1) {
g_fds.p[pid].kind = kFdProcess;
g_fds.p[pid].handle = handle;
g_fds.p[pid].flags = O_CLOEXEC;
} else {
_spunlock(&__fds_lock);
CloseHandle(handle);
return echild();
}
} else {
_spunlock(&__fds_lock);
return echild();
}
}
@ -84,8 +88,12 @@ static textwindows int sys_wait4_nt_impl(int pid, int *opt_out_wstatus,
count = 1;
} else {
count = __sample_pids(pids, handles, false);
if (!count) return echild();
if (!count) {
_spunlock(&__fds_lock);
return echild();
}
}
_spunlock(&__fds_lock);
for (;;) {
if (_check_interrupts(true, 0)) return eintr();
dwExitCode = kNtStillActive;

39
libc/intrin/asan.c
View file

@ -27,9 +27,11 @@
#include "libc/dce.h"
#include "libc/intrin/asan.internal.h"
#include "libc/intrin/asancodes.h"
#include "libc/intrin/cmpxchg.h"
#include "libc/intrin/kprintf.h"
#include "libc/intrin/lockcmpxchg.h"
#include "libc/intrin/nomultics.internal.h"
#include "libc/intrin/spinlock.h"
#include "libc/log/backtrace.internal.h"
#include "libc/log/internal.h"
#include "libc/log/libfatal.internal.h"
@ -154,7 +156,8 @@ struct ReportOriginHeap {
int z;
};
bool __asan_noreentry;
static int __asan_noreentry;
_Alignas(64) static int __asan_lock;
static struct AsanMorgue __asan_morgue;
#define __asan_unreachable() \
@ -835,30 +838,27 @@ dontdiscard __asan_die_f *__asan_report_memory_fault(void *addr, int size,
}
void *__asan_morgue_add(void *p) {
int i;
void *r;
int i, j;
for (;;) {
i = __asan_morgue.i;
j = (i + 1) & (ARRAYLEN(__asan_morgue.p) - 1);
if (_lockcmpxchg(&__asan_morgue.i, i, j)) {
r = __asan_morgue.p[i];
__asan_morgue.p[i] = p;
return r;
}
}
_spinlock_optimistic(&__asan_lock);
i = __asan_morgue.i++ & (ARRAYLEN(__asan_morgue.p) - 1);
r = __asan_morgue.p[i];
__asan_morgue.p[i] = p;
_spunlock(&__asan_lock);
return r;
}
static void __asan_morgue_flush(void) {
int i;
void *p;
_spinlock_optimistic(&__asan_lock);
for (i = 0; i < ARRAYLEN(__asan_morgue.p); ++i) {
p = __asan_morgue.p[i];
if (_lockcmpxchg(__asan_morgue.p + i, p, 0)) {
if (weaken(dlfree)) {
weaken(dlfree)(p);
}
if (weaken(dlfree)) {
weaken(dlfree)(__asan_morgue.p[i]);
}
__asan_morgue.p[i] = 0;
}
_spunlock(&__asan_lock);
}
static size_t __asan_user_size(size_t n) {
@ -1197,12 +1197,13 @@ void __asan_evil(uint8_t *addr, int size, const char *s1, const char *s2) {
struct AsanTrace tr;
__asan_rawtrace(&tr, __builtin_frame_address(0));
kprintf(
"WARNING: ASAN error during %s bad %d byte %s at %x bt %x %x %x %x %x\n",
"WARNING: ASAN %s %s bad %d byte %s at %x bt %x %x %x %x %x\n",
__asan_noreentry == gettid() ? "error during" : "multi-threaded crash",
s1, size, s2, addr, tr.p[0], tr.p[1], tr.p[2], tr.p[3], tr.p[4], tr.p[5]);
}
void __asan_report_load(uint8_t *addr, int size) {
if (_lockcmpxchg(&__asan_noreentry, false, true)) {
if (_lockcmpxchg(&__asan_noreentry, 0, gettid())) {
if (!__vforked) {
__asan_report_memory_fault(addr, size, "load")();
__asan_unreachable();
@ -1215,7 +1216,7 @@ void __asan_report_load(uint8_t *addr, int size) {
}
void __asan_report_store(uint8_t *addr, int size) {
if (_lockcmpxchg(&__asan_noreentry, false, true)) {
if (_lockcmpxchg(&__asan_noreentry, 0, gettid())) {
if (!__vforked) {
__asan_report_memory_fault(addr, size, "store")();
__asan_unreachable();

2
libc/intrin/asan.internal.h
View file

@ -16,8 +16,6 @@ struct AsanFault {
const signed char *shadow;
};
extern bool __asan_noreentry;
void __asan_unpoison(long, long);
void __asan_poison(long, long, signed char);
void __asan_verify(const void *, size_t);

6
libc/intrin/cxaatexit.c
View file

@ -19,6 +19,7 @@
#include "libc/assert.h"
#include "libc/bits/weaken.h"
#include "libc/calls/strace.internal.h"
#include "libc/intrin/spinlock.h"
#include "libc/macros.internal.h"
#include "libc/mem/mem.h"
#include "libc/nexgen32e/bsr.h"
@ -28,6 +29,8 @@
STATIC_YOINK("__cxa_finalize");
static int __cxa_lock;
/**
* Adds global destructor.
*
@ -47,6 +50,7 @@ noasan int __cxa_atexit(void *fp, void *arg, void *pred) {
unsigned i;
struct CxaAtexitBlock *b, *b2;
_Static_assert(ATEXIT_MAX == CHAR_BIT * sizeof(b->mask), "");
_spinlock(&__cxa_lock);
b = __cxa_blocks.p;
if (!b) b = __cxa_blocks.p = &__cxa_blocks.root;
if (!~b->mask) {
@ -55,6 +59,7 @@ noasan int __cxa_atexit(void *fp, void *arg, void *pred) {
b2->next = b;
__cxa_blocks.p = b = b2;
} else {
_spunlock(&__cxa_lock);
return enomem();
}
}
@ -64,5 +69,6 @@ noasan int __cxa_atexit(void *fp, void *arg, void *pred) {
b->p[i].fp = fp;
b->p[i].arg = arg;
b->p[i].pred = pred;
_spunlock(&__cxa_lock);
return 0;
}

3
libc/intrin/exit1.greg.c
View file

@ -19,7 +19,6 @@
#include "libc/calls/strace.internal.h"
#include "libc/dce.h"
#include "libc/intrin/setjmp.internal.h"
#include "libc/intrin/winthread.internal.h"
#include "libc/nt/thread.h"
#include "libc/runtime/runtime.h"
#include "libc/sysv/consts/nr.h"
@ -34,7 +33,7 @@
*/
privileged wontreturn void _Exit1(int rc) {
struct WinThread *wt;
/* STRACE("_Exit1(%d)", rc); */
STRACE("_Exit1(%d)", rc);
if (!IsWindows() && !IsMetal()) {
register long r10 asm("r10") = 0;
asm volatile("syscall"

2
libc/intrin/g_fds.c
View file

@ -25,7 +25,7 @@
STATIC_YOINK("_init_g_fds");
struct Fds g_fds;
_Alignas(64) char __fds_lock;
_Alignas(64) int __fds_lock;
textstartup void InitializeFileDescriptors(void) {
struct Fds *fds;

2
libc/intrin/gettid.greg.c
View file

@ -18,7 +18,7 @@
╚─────────────────────────────────────────────────────────────────────────────*/
#include "libc/calls/calls.h"
#include "libc/dce.h"
#include "libc/intrin/tls.h"
#include "libc/intrin/threaded.h"
#include "libc/nt/thread.h"
/**

2
libc/intrin/kprintf.greg.c
View file

@ -31,7 +31,7 @@
#include "libc/intrin/lockcmpxchg.h"
#include "libc/intrin/nomultics.internal.h"
#include "libc/intrin/spinlock.h"
#include "libc/intrin/threaded.internal.h"
#include "libc/intrin/threaded.h"
#include "libc/limits.h"
#include "libc/log/internal.h"
#include "libc/macros.internal.h"

8
libc/intrin/releasefd.c
View file

@ -20,11 +20,15 @@
#include "libc/intrin/spinlock.h"
#include "libc/macros.internal.h"
void __releasefd(int fd) {
_spinlock(&__fds_lock);
void __releasefd_unlocked(int fd) {
if (0 <= fd && fd < g_fds.n) {
g_fds.p[fd].kind = 0;
g_fds.f = MIN(fd, g_fds.f);
}
}
void __releasefd(int fd) {
_spinlock(&__fds_lock);
__releasefd_unlocked(fd);
_spunlock(&__fds_lock);
}

52
libc/intrin/spinlock.h
View file

@ -2,16 +2,26 @@
#define COSMOPOLITAN_LIBC_INTRIN_SPINLOCK_H_
#include "libc/bits/weaken.h"
#include "libc/calls/calls.h"
#include "libc/dce.h"
#include "libc/intrin/kprintf.h"
#include "libc/intrin/lockcmpxchg.h"
#include "libc/log/backtrace.internal.h"
#include "libc/log/log.h"
#include "libc/runtime/symbols.internal.h"
#if IsModeDbg() && !defined(_SPINLOCK_DEBUG)
#define _SPINLOCK_DEBUG
#endif
#if defined(_SPINLOCK_DEBUG)
#define _spinlock(lock) _spinlock_debug(lock)
#define _spinlock(lock) _spinlock_ndebug(lock)
#define _spinlock_ndebug(lock) _spinlock_optimistic(lock)
#elif defined(TINY)
#define _spinlock(lock) _spinlock_tiny(lock)
#define _spinlock(lock) _spinlock_tiny(lock)
#define _spinlock_ndebug(lock) _spinlock_tiny(lock)
#else
#define _spinlock(lock) _spinlock_optimistic(lock)
#define _spinlock(lock) _spinlock_optimistic(lock)
#define _spinlock_ndebug(lock) _spinlock_optimistic(lock)
#endif
#define _spunlock(lock) __atomic_clear(lock, __ATOMIC_RELAXED)
@ -44,22 +54,26 @@
} \
} while (0)
#define _spinlock_debug(lock) \
do { \
typeof(*(lock)) me, owner; \
me = gettid(); \
if (_trylock(lock)) { \
__atomic_load(lock, &owner, __ATOMIC_RELAXED); \
if (owner == me) { \
kprintf("%s:%d: warning: possible spinlock re-entry in %s()\n", \
__FILE__, __LINE__, __FUNCTION__); \
if (weaken(ShowBacktrace)) { \
weaken(ShowBacktrace)(2, 0); \
} \
} \
_spinlock_optimistic(lock); \
} \
*lock = me; \
#define _spinlock_debug(lock) \
do { \
typeof(*(lock)) me, owner; \
unsigned long warntries = 10000000; \
me = gettid(); \
if (!_lockcmpxchg(lock, 0, me)) { \
__atomic_load(lock, &owner, __ATOMIC_RELAXED); \
if (owner == me) { \
kprintf("%s:%d: warning: possible re-entry on %s in %s()\n", __FILE__, \
__LINE__, #lock, __FUNCTION__); \
} \
while (!_lockcmpxchg(lock, 0, me)) { \
if (!--warntries) { \
warntries = -1; \
kprintf("%s:%d: warning: possible deadlock on %s in %s()\n", \
__FILE__, __LINE__, #lock, __FUNCTION__); \
} \
__builtin_ia32_pause(); \
} \
} \
} while (0)
#endif /* COSMOPOLITAN_LIBC_INTRIN_SPINLOCK_H_ */

16
libc/intrin/threaded.h Normal file
View file

@ -0,0 +1,16 @@
#ifndef COSMOPOLITAN_LIBC_INTRIN_THREADED_H_
#define COSMOPOLITAN_LIBC_INTRIN_THREADED_H_
#if !(__ASSEMBLER__ + __LINKER__ + 0)
COSMOPOLITAN_C_START_
extern bool __threaded;
extern bool __tls_enabled;
extern unsigned __tls_index;
void *__initialize_tls(char[hasatleast 64]);
void __install_tls(char[hasatleast 64]);
char *__get_tls(void);
COSMOPOLITAN_C_END_
#endif /* !(__ASSEMBLER__ + __LINKER__ + 0) */
#endif /* COSMOPOLITAN_LIBC_INTRIN_THREADED_H_ */

11
libc/intrin/threaded.internal.h
View file

@ -1,11 +0,0 @@
#ifndef COSMOPOLITAN_LIBC_INTRIN_THREADED_INTERNAL_H_
#define COSMOPOLITAN_LIBC_INTRIN_THREADED_INTERNAL_H_
#if !(__ASSEMBLER__ + __LINKER__ + 0)
COSMOPOLITAN_C_START_
extern bool __hastls;
extern bool __threaded;
COSMOPOLITAN_C_END_
#endif /* !(__ASSEMBLER__ + __LINKER__ + 0) */
#endif /* COSMOPOLITAN_LIBC_INTRIN_THREADED_INTERNAL_H_ */

111
libc/intrin/tls.greg.c
View file

@ -17,76 +17,67 @@
│ PERFORMANCE OF THIS SOFTWARE. │
╚─────────────────────────────────────────────────────────────────────────────*/
#include "libc/assert.h"
#include "libc/calls/calls.h"
#include "libc/dce.h"
#include "libc/intrin/tls.h"
#include "libc/errno.h"
#include "libc/intrin/threaded.h"
#include "libc/nt/thread.h"
#include "libc/nt/thunk/msabi.h"
#include "libc/sysv/consts/nrlinux.h"
__msabi extern typeof(TlsFree) *const __imp_TlsFree;
__msabi extern typeof(TlsAlloc) *const __imp_TlsAlloc;
__msabi extern typeof(TlsGetValue) *const __imp_TlsGetValue;
__msabi extern typeof(TlsSetValue) *const __imp_TlsSetValue;
#define __NR_sysarch 0x000000a5
#define __NR___set_tcb 0x00000149
#define __NR__lwp_setprivate 0x0000013d
#define __NR_thread_fast_set_cthread_self 0x03000003
/**
* Assigns thread-local storage slot.
*
* This function may for instance be called at startup and the result
* can be assigned to a global static variable; from then on, all the
* threads in your application may pass that value to TlsGetValue, to
* retrieve their thread-local values.
*
* @return index on success, or -1u w/ errno
* @threadsafe
* Initializes thread information block.
*/
uint32_t TlsAlloc(void) {
return __imp_TlsAlloc();
privileged void *__initialize_tls(char tib[hasatleast 64]) {
*(intptr_t *)tib = (intptr_t)tib;
*(intptr_t *)(tib + 0x30) = (intptr_t)tib;
*(int *)(tib + 0x3c) = __errno;
return tib;
}
/**
* Releases thread-local storage slot.
* @threadsafe
* Installs thread information block on main process.
*/
bool32 TlsFree(uint32_t dwTlsIndex) {
return __imp_TlsFree(dwTlsIndex);
}
/**
* Sets value to thread-local storage slot.
*
* @param dwTlsIndex is something returned by TlsAlloc()
* @return true if successful, otherwise false
* @threadsafe
*/
bool32 TlsSetValue(uint32_t dwTlsIndex, void *lpTlsValue) {
assert(IsWindows());
if (dwTlsIndex < 64) {
asm("mov\t%1,%%gs:%0"
: "=m"(*((long *)0x1480 + dwTlsIndex))
: "r"(lpTlsValue));
return true;
privileged void __install_tls(char tib[hasatleast 64]) {
int ax, dx;
uint64_t magic;
unsigned char *p;
if (IsWindows()) {
if (!__tls_index) {
__tls_index = TlsAlloc();
}
asm("mov\t%1,%%gs:%0" : "=m"(*((long *)0x1480 + __tls_index)) : "r"(tib));
} else if (IsFreebsd()) {
asm volatile("syscall"
: "=a"(ax)
: "0"(__NR_sysarch), "D"(129), "S"(tib)
: "rcx", "r11", "memory", "cc");
} else if (IsXnu()) {
asm volatile("syscall"
: "=a"(ax)
: "0"(__NR_thread_fast_set_cthread_self),
"D"((intptr_t)tib - 0x30)
: "rcx", "r11", "memory", "cc");
} else if (IsOpenbsd()) {
asm volatile("syscall"
: "=a"(ax)
: "0"(__NR___set_tcb), "D"(tib)
: "rcx", "r11", "memory", "cc");
} else if (IsNetbsd()) {
asm volatile("syscall"
: "=a"(ax), "=d"(dx)
: "0"(__NR__lwp_setprivate), "D"(tib)
: "rcx", "r11", "memory", "cc");
} else {
return __imp_TlsSetValue(dwTlsIndex, lpTlsValue);
}
}
/**
* Retrieves value from thread-local storage slot.
*
* @param dwTlsIndex is something returned by TlsAlloc()
* @return true if successful, otherwise false
* @threadsafe
*/
void *TlsGetValue(uint32_t dwTlsIndex) {
void *lpTlsValue;
assert(IsWindows());
if (dwTlsIndex < 64) {
asm("mov\t%%gs:%1,%0"
: "=r"(lpTlsValue)
: "m"(*((long *)0x1480 + dwTlsIndex)));
return lpTlsValue;
// // this could also be written as...
// asm("movq\t%%gs:0x30,%0" : "=a"(tib));
// return (void *)tib[0x1480 / 8 + dwTlsIndex];
} else {
return __imp_TlsGetValue(dwTlsIndex);
asm volatile("syscall"
: "=a"(ax)
: "0"(__NR_linux_arch_prctl), "D"(ARCH_SET_FS), "S"(tib)
: "rcx", "r11", "memory");
}
__tls_enabled = true;
}

13
libc/intrin/tls.h
View file

@ -1,13 +0,0 @@
#ifndef COSMOPOLITAN_LIBC_INTRIN_TLS_H_
#define COSMOPOLITAN_LIBC_INTRIN_TLS_H_
#if !(__ASSEMBLER__ + __LINKER__ + 0)
COSMOPOLITAN_C_START_
uint32_t TlsAlloc(void);
bool32 TlsFree(uint32_t);
bool32 TlsSetValue(uint32_t, void *);
void *TlsGetValue(uint32_t);
COSMOPOLITAN_C_END_
#endif /* !(__ASSEMBLER__ + __LINKER__ + 0) */
#endif /* COSMOPOLITAN_LIBC_INTRIN_TLS_H_ */

38
libc/intrin/winthread.c
View file

@ -1,38 +0,0 @@
/*-*- mode:c;indent-tabs-mode:nil;c-basic-offset:2;tab-width:8;coding:utf-8 -*-│
│vi: set net ft=c ts=2 sts=2 sw=2 fenc=utf-8 :vi│
╞══════════════════════════════════════════════════════════════════════════════╡
│ Copyright 2022 Justine Alexandra Roberts Tunney │
│ │
│ Permission to use, copy, modify, and/or distribute this software for │
│ any purpose with or without fee is hereby granted, provided that the │
│ above copyright notice and this permission notice appear in all copies. │
│ │
│ THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL │
│ WARRANTIES WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED │
│ WARRANTIES OF MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE │
│ AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL │
│ DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR │
│ PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER │
│ TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR │
│ PERFORMANCE OF THIS SOFTWARE. │
╚─────────────────────────────────────────────────────────────────────────────*/
#include "libc/dce.h"
#include "libc/intrin/tls.h"
#include "libc/intrin/winthread.internal.h"
/**
* @fileoverview TLS slot for clone() win32 polyfill.
*/
int __winthread;
static textstartup void __winthread_init(void) {
if (IsWindows()) {
__winthread = TlsAlloc();
TlsSetValue(__winthread, 0);
}
}
const void *const __winthread_ctor[] initarray = {
__winthread_init,
};

24
libc/intrin/winthread.internal.h
View file

@ -1,24 +0,0 @@
#ifndef COSMOPOLITAN_LIBC_RUNTIME_WINTHREAD_INTERNAL_H_
#define COSMOPOLITAN_LIBC_RUNTIME_WINTHREAD_INTERNAL_H_
#include "libc/intrin/tls.h"
#include "libc/runtime/runtime.h"
#if !(__ASSEMBLER__ + __LINKER__ + 0)
COSMOPOLITAN_C_START_
struct WinThread {
uint32_t tid;
int flags;
int *ctid;
int (*func)(void *);
void *arg;
};
extern int __winthread;
static inline struct WinThread *GetWinThread(void) {
return TlsGetValue(__winthread);
}
COSMOPOLITAN_C_END_
#endif /* !(__ASSEMBLER__ + __LINKER__ + 0) */
#endif /* COSMOPOLITAN_LIBC_RUNTIME_WINTHREAD_INTERNAL_H_ */

2
libc/log/vflogf.c
View file

@ -41,7 +41,7 @@
#define kNontrivialSize (8 * 1000 * 1000)
static struct timespec vflogf_ts;
_Alignas(64) static char vflogf_lock;
_Alignas(64) static int vflogf_lock;
/**
* Takes corrective action if logging is on the fritz.

21
libc/nexgen32e/hastls.c
View file

@ -1,21 +0,0 @@
/*-*- mode:c;indent-tabs-mode:nil;c-basic-offset:2;tab-width:8;coding:utf-8 -*-│
│vi: set net ft=c ts=2 sts=2 sw=2 fenc=utf-8 :vi│
╞══════════════════════════════════════════════════════════════════════════════╡
│ Copyright 2022 Justine Alexandra Roberts Tunney │
│ │
│ Permission to use, copy, modify, and/or distribute this software for │
│ any purpose with or without fee is hereby granted, provided that the │
│ above copyright notice and this permission notice appear in all copies. │
│ │
│ THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL │
│ WARRANTIES WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED │
│ WARRANTIES OF MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE │
│ AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL │
│ DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR │
│ PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER │
│ TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR │
│ PERFORMANCE OF THIS SOFTWARE. │
╚─────────────────────────────────────────────────────────────────────────────*/
#include "libc/intrin/threaded.internal.h"
bool __hastls;

1
libc/nexgen32e/nexgen32e.mk
View file

@ -42,7 +42,6 @@ $(LIBC_NEXGEN32E_A).pkg: \
$(LIBC_NEXGEN32E_A_OBJS) \
$(foreach x,$(LIBC_NEXGEN32E_A_DIRECTDEPS),$($(x)_A).pkg)
o/$(MODE)/libc/nexgen32e/hastls.o \
o/$(MODE)/libc/nexgen32e/threaded.o: \
OVERRIDE_CFLAGS += \
$(NO_MAGIC) \

4
libc/nexgen32e/threaded.c
View file

@ -16,6 +16,8 @@
│ TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR │
│ PERFORMANCE OF THIS SOFTWARE. │
╚─────────────────────────────────────────────────────────────────────────────*/
#include "libc/intrin/threaded.internal.h"
#include "libc/intrin/threaded.h"
bool __threaded;
bool __tls_enabled;
unsigned __tls_index;

4
libc/nt/kernel32/TlsAlloc.s
View file

@ -2,7 +2,7 @@
.imp kernel32,__imp_TlsAlloc,TlsAlloc,0
.text.windows
__TlsAlloc:
TlsAlloc:
push %rbp
mov %rsp,%rbp
.profilable
@ -10,5 +10,5 @@ __TlsAlloc:
call *__imp_TlsAlloc(%rip)
leave
ret
.endfn __TlsAlloc,globl
.endfn TlsAlloc,globl
.previous

4
libc/nt/kernel32/TlsFree.s
View file

@ -2,7 +2,7 @@
.imp kernel32,__imp_TlsFree,TlsFree,0
.text.windows
__TlsFree:
TlsFree:
push %rbp
mov %rsp,%rbp
.profilable
@ -11,5 +11,5 @@ __TlsFree:
call *__imp_TlsFree(%rip)
leave
ret
.endfn __TlsFree,globl
.endfn TlsFree,globl
.previous

4
libc/nt/kernel32/TlsGetValue.s
View file

@ -2,7 +2,7 @@
.imp kernel32,__imp_TlsGetValue,TlsGetValue,0
.text.windows
__TlsGetValue:
TlsGetValue:
push %rbp
mov %rsp,%rbp
.profilable
@ -11,5 +11,5 @@ __TlsGetValue:
call *__imp_TlsGetValue(%rip)
leave
ret
.endfn __TlsGetValue,globl
.endfn TlsGetValue,globl
.previous

4
libc/nt/kernel32/TlsSetValue.s
View file

@ -2,11 +2,11 @@
.imp kernel32,__imp_TlsSetValue,TlsSetValue,0
.text.windows
__TlsSetValue:
TlsSetValue:
push %rbp
mov %rsp,%rbp
.profilable
mov __imp_TlsSetValue(%rip),%rax
jmp __sysv2nt
.endfn __TlsSetValue,globl
.endfn TlsSetValue,globl
.previous

10
libc/nt/master.sh
View file

@ -302,8 +302,8 @@ imp 'EnumerateLocalComputerNamesA' EnumerateLocalComputerNamesA kernel32 3
imp 'EraseTape' EraseTape kernel32 352
imp 'EscapeCommFunction' EscapeCommFunction kernel32 0
imp 'ExecuteUmsThread' ExecuteUmsThread kernel32 354
imp 'ExitThread' ExitThread kernel32 0 1
imp 'ExitProcess' ExitProcess kernel32 0 1 # a.k.a. RtlExitUserProcess
imp 'ExitThread' ExitThread kernel32 0 1
imp 'ExitVDM' ExitVDM kernel32 357
imp 'ExpandEnvironmentStrings' ExpandEnvironmentStringsW kernel32 0
imp 'ExpandEnvironmentStringsA' ExpandEnvironmentStringsA kernel32 0
@ -1225,6 +1225,10 @@ imp 'TermsrvSetValueKey' TermsrvSetValueKey kernel32 1441
imp 'TermsrvSyncUserIniFileExt' TermsrvSyncUserIniFileExt kernel32 1442
imp 'Thread32First' Thread32First kernel32 1443
imp 'Thread32Next' Thread32Next kernel32 1444
imp 'TlsAlloc' TlsAlloc kernel32 0 0
imp 'TlsFree' TlsFree kernel32 0 1
imp 'TlsGetValue' TlsGetValue kernel32 0 1
imp 'TlsSetValue' TlsSetValue kernel32 0 2
imp 'Toolhelp32ReadProcessMemory' Toolhelp32ReadProcessMemory kernel32 1449
imp 'TransactNamedPipe' TransactNamedPipe kernel32 0 7
imp 'TransmitCommChar' TransmitCommChar kernel32 0
@ -1364,10 +1368,6 @@ imp '__ReOpenFile' ReOpenFile kernel32 0 4 # TODO(jart): 6.2 and highe
imp '__RemoveDirectory' RemoveDirectoryW kernel32 0 1
imp '__SetCurrentDirectory' SetCurrentDirectoryW kernel32 0 1
imp '__TerminateProcess' TerminateProcess kernel32 0 2
imp '__TlsAlloc' TlsAlloc kernel32 0 0
imp '__TlsFree' TlsFree kernel32 0 1
imp '__TlsGetValue' TlsGetValue kernel32 0 1
imp '__TlsSetValue' TlsSetValue kernel32 0 2
imp '__UnmapViewOfFile' UnmapViewOfFile kernel32 0 1
imp '__VirtualProtect' VirtualProtect kernel32 0 4
imp '__WaitForMultipleObjects' WaitForMultipleObjects kernel32 0 4

5
libc/nt/thread.h
View file

@ -57,6 +57,11 @@ bool32 CancelSynchronousIo(int64_t hThread);
bool32 CancelIo(int64_t hFile);
bool32 CancelIoEx(int64_t hFile, struct NtOverlapped *opt_lpOverlapped);
uint32_t TlsAlloc(void);
bool32 TlsFree(uint32_t);
bool32 TlsSetValue(uint32_t, void *);
void *TlsGetValue(uint32_t);
#if ShouldUseMsabiAttribute()
#include "libc/nt/thunk/thread.inc"
#endif /* ShouldUseMsabiAttribute() */

248
libc/runtime/clone.greg.c
View file

@ -26,9 +26,7 @@
#include "libc/intrin/asan.internal.h"
#include "libc/intrin/kprintf.h"
#include "libc/intrin/spinlock.h"
#include "libc/intrin/threaded.internal.h"
#include "libc/intrin/tls.h"
#include "libc/intrin/winthread.internal.h"
#include "libc/intrin/threaded.h"
#include "libc/nt/runtime.h"
#include "libc/nt/thread.h"
#include "libc/nt/thunk/msabi.h"
@ -46,58 +44,22 @@ STATIC_YOINK("gettid"); // for kprintf()
#define __NR_clone_linux 56
#define __NR__lwp_create 309
#define __NR_getcontext_netbsd 307
#define __NR__lwp_setprivate 317
#define __NR_bsdthread_create 0x02000168
#define __NR_thread_fast_set_cthread_self 0x03000003
#define __NR_sysarch 0x000000a5
#define __NR___set_tcb 0x00000149
#define PTHREAD_START_CUSTOM_XNU 0x01000000
#define LWP_DETACHED 0x00000040
#define LWP_SUSPENDED 0x00000080
char __tls[512];
int __errno_global;
extern int __errno_index;
static char tibdefault[64];
privileged void __setup_tls(void) {
int ax, dx;
uint64_t magic;
unsigned char *p;
*(intptr_t *)__tls = (intptr_t)__tls;
*(intptr_t *)(__tls + 0x30) = (intptr_t)__tls;
*(int *)(__tls + 0x3c) = __errno;
if (IsWindows()) {
__errno_index = TlsAlloc();
TlsSetValue(__errno_index, (void *)(intptr_t)__errno);
} else if (IsLinux()) {
asm volatile("syscall"
: "=a"(ax)
: "0"(__NR_linux_arch_prctl), "D"(ARCH_SET_FS), "S"(__tls)
: "rcx", "r11", "memory");
} else if (IsFreebsd()) {
asm volatile("syscall"
: "=a"(ax)
: "0"(__NR_sysarch), "D"(129), "S"(__tls)
: "rcx", "r11", "memory", "cc");
} else if (IsXnu()) {
asm volatile("syscall"
: "=a"(ax)
: "0"(__NR_thread_fast_set_cthread_self),
"D"((intptr_t)__tls - 0x30)
: "rcx", "r11", "memory", "cc");
} else if (IsOpenbsd()) {
asm volatile("syscall"
: "=a"(ax)
: "0"(__NR___set_tcb), "D"(__tls)
: "rcx", "r11", "memory", "cc");
} else if (IsNetbsd()) {
asm volatile("syscall"
: "=a"(ax), "=d"(dx)
: "0"(__NR__lwp_setprivate), "D"(__tls)
: "rcx", "r11", "memory", "cc");
}
__hastls = true;
}
struct WinThread {
uint32_t tid;
int flags;
int *ctid;
void *tls;
int (*func)(void *);
void *arg;
};
uint32_t WinThreadThunk(void *warg);
asm(".section\t.text.windows,\"ax\",@progbits\n\t"
@ -115,20 +77,19 @@ __attribute__((__used__, __no_reorder__))
static textwindows wontreturn void
WinThreadMain(struct WinThread *wt) {
int rc;
if (wt->flags & CLONE_SETTLS) {
TlsSetValue(__tls_index, wt->tls);
}
if (wt->flags & CLONE_CHILD_SETTID) {
*wt->ctid = wt->tid;
}
rc = wt->func(wt->arg);
if (wt->flags & CLONE_CHILD_CLEARTID) {
*wt->ctid = 0;
}
_Exit1(rc);
}
static textwindows int CloneWindows(int (*func)(void *), char *stk,
size_t stksz, int flags, void *arg,
int *ptid, void *tls, size_t tlssz,
int *ctid) {
void *tls, size_t tlssz, int *ctid) {
int64_t h;
struct WinThread *wt;
wt = (struct WinThread *)(((intptr_t)(stk + stksz) -
@ -138,14 +99,11 @@ static textwindows int CloneWindows(int (*func)(void *), char *stk,
wt->ctid = ctid;
wt->func = func;
wt->arg = arg;
wt->tls = tls;
if ((h = CreateThread(0, 0, WinThreadThunk, wt, 0, &wt->tid))) {
CloseHandle(h);
if (flags & CLONE_PARENT_SETTID) {
*ptid = wt->tid;
}
return wt->tid;
} else {
__releasefd(wt->tid);
return -1;
}
}
@ -179,14 +137,11 @@ XnuThreadMain(void *pthread, int tid, int (*func)(void *arg), void *arg,
*(int *)sp[2] = tid;
}
rc = func(arg);
if (sp[4] & CLONE_CHILD_CLEARTID) {
*(int *)sp[2] = 0;
}
_Exit1(rc);
}
static int CloneXnu(int (*fn)(void *), char *stk, size_t stksz, int flags,
void *arg, int *ptid, void *tls, size_t tlssz, int *ctid) {
void *arg, void *tls, size_t tlssz, int *ctid) {
int rc;
bool failed;
intptr_t *sp;
@ -212,9 +167,6 @@ static int CloneXnu(int (*fn)(void *), char *stk, size_t stksz, int flags,
_seizelock(sp); // TODO: How can we get the tid without locking?
if ((rc = bsdthread_create(fn, arg, sp, 0, PTHREAD_START_CUSTOM_XNU)) != -1) {
_spinlock(sp);
if (flags & CLONE_PARENT_SETTID) {
*ptid = sp[1];
}
rc = sp[1];
}
return rc;
@ -236,15 +188,11 @@ FreebsdThreadMain(intptr_t *sp) {
*(int *)sp[2] = sp[4];
}
rc = ((int (*)(intptr_t))sp[0])(sp[1]);
if (sp[3] & CLONE_CHILD_CLEARTID) {
*(int *)sp[2] = 0;
}
_Exit1(rc);
}
static int CloneFreebsd(int (*func)(void *), char *stk, size_t stksz, int flags,
void *arg, int *ptid, void *tls, size_t tlssz,
int *ctid) {
void *arg, void *tls, size_t tlssz, int *ctid) {
int ax;
bool failed;
int64_t tid;
@ -270,15 +218,11 @@ static int CloneFreebsd(int (*func)(void *), char *stk, size_t stksz, int flags,
: CFLAG_CONSTRAINT(failed), "=a"(ax)
: "1"(__NR_thr_new), "D"(&params), "S"(sizeof(params))
: "rcx", "rdx", "r8", "r9", "r10", "r11", "memory");
if (!failed) {
if (flags & CLONE_PARENT_SETTID) {
*ptid = tid;
}
return tid;
} else {
if (failed) {
errno = ax;
return -1;
tid = -1;
}
return tid;
}
struct __tfork {
@ -313,15 +257,11 @@ static privileged wontreturn void
OpenbsdThreadMain(intptr_t *sp) {
int rc;
rc = ((int (*)(intptr_t))sp[0])(sp[1]);
if (sp[3] & CLONE_CHILD_CLEARTID) {
*(int *)sp[2] = 0;
}
_Exit1(rc);
}
static int CloneOpenbsd(int (*func)(void *), char *stk, size_t stksz, int flags,
void *arg, int *ptid, void *tls, size_t tlssz,
int *ctid) {
void *arg, void *tls, size_t tlssz, int *ctid) {
int tid;
intptr_t *sp;
struct __tfork params;
@ -333,11 +273,7 @@ static int CloneOpenbsd(int (*func)(void *), char *stk, size_t stksz, int flags,
params.tf_stack = sp;
params.tf_tcb = flags & CLONE_SETTLS ? tls : 0;
params.tf_tid = flags & CLONE_CHILD_SETTID ? ctid : 0;
if ((tid = __tfork(&params, sizeof(params), sp)) > 0) {
if (flags & CLONE_PARENT_SETTID) {
*ptid = tid;
}
} else {
if ((tid = __tfork(&params, sizeof(params), sp)) < 0) {
errno = -tid;
tid = -1;
}
@ -351,15 +287,11 @@ static wontreturn void NetbsdThreadMain(void *arg, int (*func)(void *arg),
*ctid = *tid;
}
rc = func(arg);
if (flags & CLONE_CHILD_CLEARTID) {
*ctid = 0;
}
_Exit1(rc);
}
static int CloneNetbsd(int (*func)(void *), char *stk, size_t stksz, int flags,
void *arg, int *ptid, void *tls, size_t tlssz,
int *ctid) {
void *arg, void *tls, size_t tlssz, int *ctid) {
// NetBSD has its own clone() and it works, but it's technically a
// second-class API, intended to help Linux folks migrate to this!
// We put it on the thread's stack, to avoid locking this function
@ -414,9 +346,6 @@ static int CloneNetbsd(int (*func)(void *), char *stk, size_t stksz, int flags,
: "1"(__NR__lwp_create), "D"(ctx), "S"(LWP_DETACHED), "2"(tid)
: "rcx", "r11", "memory");
if (!failed) {
if (flags & CLONE_PARENT_SETTID) {
*ptid = *tid;
}
return *tid;
} else {
errno = ax;
@ -453,49 +382,65 @@ static int CloneLinux(int (*func)(void *), char *stk, size_t stksz, int flags,
* Creates thread.
*
* Threads are created in a detached manner. They currently can't be
* synchronized using wait() and posix signals. Threads created by this
* synchronized using wait() or posix signals. Threads created by this
* function should be synchronized using shared memory operations.
*
* Any memory that's required by this system call wrapper is allocated
* to the top of your stack. This is normally about 64 bytes, although
* on NetBSD it's currently 800.
*
* Your function is called from within the stack you specify. A return
* address is pushed onto your stack, that causes returning to jump to
* _Exit1() which terminates the thread. Even though the callback says
* it supports a return code, that'll only work on Linux and Windows.
*
* The `tls` parameter is for thread-local storage. If you specify this
* then clone() will implicitly rewire libc (e.g. errno) to use TLS:
*
* static char tib[64];
* __initialize_tls(tib);
* __install_tls(tib);
*
* If you want a main process TLS size that's larger call it manually.
* Once you've done the above and/or started creating your own threads
* you'll be able to access your `tls` thread information block, using
*
* char *p = __get_tls();
* printf("errno is %d\n", *(int *)(p + 0x3c));
*
* This function follows the same ABI convention as the Linux userspace
* libraries, with a few small changes. The varargs has been removed to
* help prevent broken code, and the stack size and tls size parameters
* are introduced for compatibility with FreeBSD.
*
* To keep this system call lightweight, only the thread creation use
* case is polyfilled across platforms. For example, if you want fork
* that works on OpenBSD for example, don't do it with clone(SIGCHLD)
* and please just call fork(). Even if you do that on Linux, it will
* effectively work around libc features like atfork(), so that means
* other calls like getpid() may return incorrect values.
*
* @param func is your callback function
* @param stk points to the bottom of a caller allocated stack, which
* must be null when fork() and vfork() equivalent flags are used
* and furthermore this must be mmap()'d using MAP_STACK in order
* to work on OpenBSD
* @param stksz is the size of that stack in bytes which must be zero
* if the fork() or vfork() equivalent flags are used it's highly
* recommended that this value be GetStackSize(), or else kprintf
* and other runtime services providing memory safety can't do as
* good and quick of a job; this value must be 16-aligned plus it
* must be at minimum 4096 bytes in size
* @param flags usually has one of
* - `SIGCHLD` will delegate to fork()
* - `CLONE_VFORK|CLONE_VM|SIGCHLD` means vfork()
* must be allocated via mmap() using the MAP_STACK flag, or else
* you won't get optimal performance and it won't work on OpenBSD
* @param stksz is the size of that stack in bytes, we recommend that
* that this be set to GetStackSize() or else memory safety tools
* like kprintf() can't do as good and quick of a job; this value
* must be 16-aligned plus it must be at least 4192 bytes in size
* and it's advised to have the bottom-most page, be a guard page
* @param flags should have:
* - `CLONE_THREAD|CLONE_VM|CLONE_FS|CLONE_FILES|CLONE_SIGHAND`
* as part high bytes, and the low order byte may optionally contain
* a signal e.g. SIGCHLD, to enable parent notification on terminate
* although the signal isn't supported on non-Linux and non-NetBSD
* at the moment; 'flags' may optionally bitwise or the following:
* - `CLONE_PARENT_SETTID` is needed for `ctid` should be set
* - `CLONE_CHILD_SETTID` is needed for `ptid` should be set
* - `CLONE_SETTLS` is needed to set `%fs` segment to `tls`
* and may optionally bitwise any of the following:
* - `CLONE_CHILD_SETTID` is needed too if you use `ctid`
* - `CLONE_SETTLS` is needed too if you set `tls`
* @param arg will be passed to your callback
* @param ptid lets the parent receive the child thread id;
* this parameter is ignored if `CLONE_PARENT_SETTID` is not set
* @param tls may be used to set the thread local storage segment;
* this parameter is ignored if `CLONE_SETTLS` is not set
* @param tlssz is the size of tls in bytes
* @param ctid lets the child receive its thread id;
* this parameter is ignored if `CLONE_CHILD_SETTID` is not set
* @return tid on success and 0 to the child, or -1 w/ errno
* @param tlssz is the size of tls in bytes which must be at least 64
* @param ctid lets the child receive its thread id without having to
* call gettid() and is ignored if `CLONE_CHILD_SETTID` isn't set
* @return tid of child on success, or -1 w/ errno
* @threadsafe
*/
int clone(int (*func)(void *), void *stk, size_t stksz, int flags, void *arg,
@ -503,11 +448,11 @@ int clone(int (*func)(void *), void *stk, size_t stksz, int flags, void *arg,
int rc;
__threaded = true;
if (tls && !__hastls) {
__setup_tls();
if (tls && !__tls_enabled) {
__initialize_tls(tibdefault);
__install_tls(tibdefault);
}
// verify memory is kosher
if (IsAsan() &&
((stksz > PAGESIZE &&
!__asan_is_valid((char *)stk + PAGESIZE, stksz - PAGESIZE)) ||
@ -518,54 +463,27 @@ int clone(int (*func)(void *), void *stk, size_t stksz, int flags, void *arg,
((flags & CLONE_CHILD_SETTID) &&
!__asan_is_valid(ctid, sizeof(*ctid))))) {
rc = efault();
}
// delegate to bona fide clone()
else if (IsLinux()) {
} else if (!IsTiny() &&
(((flags & CLONE_VM) && (stksz < PAGESIZE || (stksz & 15))) ||
((flags & CLONE_SETTLS) && (tlssz < 64 || (tlssz & 7))))) {
rc = einval();
} else if (IsLinux()) {
rc = CloneLinux(func, stk, stksz, flags, arg, ptid, tls, tlssz, ctid);
}
// polyfill fork() and vfork() use cases on platforms without clone()
else if ((SupportsWindows() || SupportsBsd()) &&
flags == (CLONE_VFORK | CLONE_VM | SIGCHLD)) {
if (IsTiny()) {
rc = einval();
} else if (!arg && !stksz) {
return vfork(); // don't log clone()
} else {
rc = einval();
}
} else if ((SupportsWindows() || SupportsBsd()) && flags == SIGCHLD) {
if (IsTiny()) {
rc = eopnotsupp();
} else if (!arg && !stksz) {
return fork(); // don't log clone()
} else {
rc = einval();
}
}
// we now assume we're creating a thread
// these platforms can't do signals the way linux does
else if (!IsTiny() && ((stksz < PAGESIZE || (stksz & 15)) ||
(flags & ~(CLONE_SETTLS | CLONE_PARENT_SETTID |
CLONE_CHILD_SETTID)) !=
(CLONE_THREAD | CLONE_VM | CLONE_FS | CLONE_FILES |
CLONE_SIGHAND))) {
} else if (!IsTiny() && (flags & ~(CLONE_SETTLS | CLONE_PARENT_SETTID |
CLONE_CHILD_SETTID)) !=
(CLONE_THREAD | CLONE_VM | CLONE_FS |
CLONE_FILES | CLONE_SIGHAND)) {
rc = einval();
} else if (IsXnu()) {
rc = CloneXnu(func, stk, stksz, flags, arg, ptid, tls, tlssz, ctid);
rc = CloneXnu(func, stk, stksz, flags, arg, tls, tlssz, ctid);
} else if (IsFreebsd()) {
rc = CloneFreebsd(func, stk, stksz, flags, arg, ptid, tls, tlssz, ctid);
rc = CloneFreebsd(func, stk, stksz, flags, arg, tls, tlssz, ctid);
} else if (IsNetbsd()) {
rc = CloneNetbsd(func, stk, stksz, flags, arg, ptid, tls, tlssz, ctid);
rc = CloneNetbsd(func, stk, stksz, flags, arg, tls, tlssz, ctid);
} else if (IsOpenbsd()) {
rc = CloneOpenbsd(func, stk, stksz, flags, arg, ptid, tls, tlssz, ctid);
}
// These platforms can't do segment registers like linux does
else if (IsWindows()) {
rc = CloneWindows(func, stk, stksz, flags, arg, ptid, tls, tlssz, ctid);
rc = CloneOpenbsd(func, stk, stksz, flags, arg, tls, tlssz, ctid);
} else if (IsWindows()) {
rc = CloneWindows(func, stk, stksz, flags, arg, tls, tlssz, ctid);
} else {
rc = enosys();
}

4
libc/runtime/getsymboltable.greg.c
View file

@ -20,6 +20,7 @@
#include "libc/bits/bits.h"
#include "libc/bits/weaken.h"
#include "libc/calls/strace.internal.h"
#include "libc/intrin/spinlock.h"
#include "libc/macros.internal.h"
#include "libc/runtime/runtime.h"
#include "libc/runtime/symbols.internal.h"
@ -29,6 +30,7 @@
#include "libc/zip.h"
#include "libc/zipos/zipos.internal.h"
static char g_lock;
static struct SymbolTable *g_symtab;
/**
@ -118,6 +120,7 @@ static struct SymbolTable *GetSymbolTableFromElf(void) {
*/
struct SymbolTable *GetSymbolTable(void) {
struct Zipos *z;
if (_trylock(&g_lock)) return 0;
if (!g_symtab && !__isworker) {
if (weaken(__zipos_get) && (z = weaken(__zipos_get)())) {
if ((g_symtab = GetSymbolTableFromZip(z))) {
@ -131,6 +134,7 @@ struct SymbolTable *GetSymbolTable(void) {
g_symtab = GetSymbolTableFromElf();
}
}
_spunlock(&g_lock);
return g_symtab;
}

2
libc/runtime/memtrack.internal.h
View file

@ -46,7 +46,7 @@ struct MemoryIntervals {
size_t i, n;
struct MemoryInterval *p;
struct MemoryInterval s[OPEN_MAX];
_Alignas(64) char lock;
_Alignas(64) int lock;
};
extern hidden struct MemoryIntervals _mmi;

3
libc/sock/closesocket-nt.c
View file

@ -16,6 +16,9 @@
│ TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR │
│ PERFORMANCE OF THIS SOFTWARE. │
╚─────────────────────────────────────────────────────────────────────────────*/
#include "libc/bits/weaken.h"
#include "libc/intrin/kprintf.h"
#include "libc/log/backtrace.internal.h"
#include "libc/mem/mem.h"
#include "libc/sock/internal.h"

7
libc/sock/kntwsadata.c
View file

@ -20,6 +20,7 @@
#include "libc/calls/calls.h"
#include "libc/calls/strace.internal.h"
#include "libc/dce.h"
#include "libc/intrin/spinlock.h"
#include "libc/mem/mem.h"
#include "libc/nt/runtime.h"
#include "libc/nt/winsock.h"
@ -40,12 +41,6 @@ hidden struct NtWsaData kNtWsaData;
static textwindows void WinSockCleanup(void) {
int i, rc;
NTTRACE("WinSockCleanup()");
for (i = g_fds.n; i--;) {
if (g_fds.p[i].kind == kFdSocket) {
close(i);
}
}
// TODO(jart): Check WSACleanup() result code
rc = WSACleanup();
NTTRACE("WSACleanup() → %d% lm", rc);
}

3
libc/sock/socket-nt.c
View file

@ -17,6 +17,7 @@
│ PERFORMANCE OF THIS SOFTWARE. │
╚─────────────────────────────────────────────────────────────────────────────*/
#include "libc/calls/internal.h"
#include "libc/intrin/spinlock.h"
#include "libc/mem/mem.h"
#include "libc/nt/enum/fileflagandattributes.h"
#include "libc/nt/iphlpapi.h"
@ -61,11 +62,13 @@ textwindows int sys_socket_nt(int family, int type, int protocol) {
sockfd->family = family;
sockfd->type = truetype;
sockfd->protocol = protocol;
_spinlock(&__fds_lock);
g_fds.p[fd].kind = kFdSocket;
g_fds.p[fd].flags = oflags;
g_fds.p[fd].mode = 0140666;
g_fds.p[fd].handle = h;
g_fds.p[fd].extra = (uintptr_t)sockfd;
_spunlock(&__fds_lock);
return fd;
} else {
__releasefd(fd);

5
libc/sock/socketpair-nt.c
View file

@ -16,6 +16,7 @@
│ TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR │
│ PERFORMANCE OF THIS SOFTWARE. │
╚─────────────────────────────────────────────────────────────────────────────*/
#include "libc/intrin/spinlock.h"
#include "libc/nt/createfile.h"
#include "libc/nt/enum/accessmask.h"
#include "libc/nt/enum/creationdisposition.h"
@ -74,6 +75,8 @@ textwindows int sys_socketpair_nt(int family, int type, int proto, int sv[2]) {
return -1;
}
_spinlock(&__fds_lock);
g_fds.p[reader].kind = kFdFile;
g_fds.p[reader].flags = oflags;
g_fds.p[reader].mode = 0140444;
@ -84,6 +87,8 @@ textwindows int sys_socketpair_nt(int family, int type, int proto, int sv[2]) {
g_fds.p[writer].mode = 0140222;
g_fds.p[writer].handle = h1;
_spunlock(&__fds_lock);
sv[0] = reader;
sv[1] = writer;
return 0;

2
libc/stdio/fflush.internal.h
View file

@ -10,7 +10,7 @@ struct StdioFlushHandles {
};
struct StdioFlush {
char lock;
int lock;
struct StdioFlushHandles handles;
FILE *handles_initmem[8];
};

2
libc/stdio/stdio.h
View file

@ -24,7 +24,7 @@ typedef struct FILE {
uint32_t size; /* 0x20 */
uint32_t nofree; /* 0x24 */
int pid; /* 0x28 */
char lock; /* 0x2c */
int lock; /* 0x2c */
char *getln; /* 0x30 */
} FILE;

40
libc/sysv/errno.greg.c
View file

@ -19,7 +19,8 @@
#include "libc/bits/weaken.h"
#include "libc/calls/calls.h"
#include "libc/dce.h"
#include "libc/intrin/threaded.internal.h"
#include "libc/intrin/threaded.h"
#include "libc/nt/thread.h"
/**
* Global variable for last error.
@ -34,22 +35,29 @@
* @see __errno_location() stable abi
*/
errno_t __errno;
int __errno_index;
privileged nocallersavedregisters errno_t *(__errno_location)(void) {
char *tib;
if (!__hastls) {
return &__errno;
} else if (IsLinux() || IsFreebsd() || IsNetbsd() || IsOpenbsd()) {
asm("mov\t%%fs:0,%0" : "=a"(tib));
return (errno_t *)(tib + 0x3c);
} else if (IsXnu()) {
asm("mov\t%%gs:0x30,%0" : "=a"(tib));
return (errno_t *)(tib + 0x3c);
} else if (IsWindows()) {
asm("mov\t%%gs:0x30,%0" : "=a"(tib));
return (errno_t *)(tib + 0x1480 + __errno_index * 8);
/**
* Returns address of thread information block.
* @see __install_tls()
* @see clone()
*/
privileged nocallersavedregisters char *__get_tls(void) {
char *tib, *linear = (char *)0x30;
if (IsLinux() || IsFreebsd() || IsNetbsd() || IsOpenbsd()) {
asm("mov\t%%fs:(%1),%0" : "=a"(tib) : "r"(linear));
} else {
return &__errno;
asm("mov\t%%gs:(%1),%0" : "=a"(tib) : "r"(linear));
if (IsWindows()) tib = *(char **)(tib + 0x1480 + __tls_index * 8);
}
return tib;
}
/**
* Returns address of errno variable.
* @see __initialize_tls()
* @see __install_tls()
*/
privileged nocallersavedregisters errno_t *(__errno_location)(void) {
if (!__tls_enabled) return &__errno;
return (errno_t *)(__get_tls() + 0x3c);
}

1
libc/testlib/testmain.c
View file

@ -153,6 +153,7 @@ noasan int main(int argc, char *argv[]) {
__log_level = kLogInfo;
GetOpts(argc, argv);
setenv("GDB", "", true);
GetSymbolTable();
// normalize this process
FixIrregularFds();

2
libc/time/localtime.c
View file

@ -42,7 +42,7 @@ STATIC_YOINK("usr/share/zoneinfo/UTC");
** POSIX-style TZ environment variable handling from Guy Harris.
*/
_Alignas(64) static char locallock;
_Alignas(64) static int locallock;
static int lock(void) {
_spinlock(&locallock);