Improve threading support further

This commit is contained in:
Justine Tunney 2022-05-17 04:14:28 -07:00
parent 8bfb70ca3f
commit ce71677156
61 changed files with 882 additions and 747 deletions

View file

@ -20,6 +20,8 @@
#include "libc/calls/calls.h"
#include "libc/calls/internal.h"
#include "libc/calls/strace.internal.h"
#include "libc/intrin/kprintf.h"
#include "libc/intrin/spinlock.h"
#include "libc/macros.internal.h"
#include "libc/sock/internal.h"
#include "libc/sysv/errfuns.h"
@ -46,6 +48,7 @@
*/
int close(int fd) {
int rc;
_spinlock(&__fds_lock);
if (fd == -1) {
rc = 0;
} else if (fd < 0) {
@ -74,9 +77,10 @@ int close(int fd) {
}
}
if (!__vforked) {
__releasefd(fd);
__releasefd_unlocked(fd);
}
}
_spunlock(&__fds_lock);
STRACE("%s(%d) → %d% m", "close", fd, rc);
return rc;
}

View file

@ -20,6 +20,7 @@
#include "libc/bits/weaken.h"
#include "libc/calls/calls.h"
#include "libc/calls/internal.h"
#include "libc/intrin/spinlock.h"
#include "libc/mem/mem.h"
#include "libc/nt/files.h"
#include "libc/nt/runtime.h"
@ -32,25 +33,37 @@
* Implements dup(), dup2(), dup3(), and F_DUPFD for Windows.
*/
textwindows int sys_dup_nt(int oldfd, int newfd, int flags, int start) {
int64_t proc, handle;
int64_t rc, proc, handle;
// validate the api usage
if (oldfd < 0) return einval();
if (flags & ~O_CLOEXEC) return einval();
_spinlock(&__fds_lock);
if (oldfd >= g_fds.n ||
(g_fds.p[oldfd].kind != kFdFile && g_fds.p[oldfd].kind != kFdSocket &&
g_fds.p[oldfd].kind != kFdConsole)) {
_spunlock(&__fds_lock);
return ebadf();
}
// allocate a new file descriptor
if (newfd == -1) {
if ((newfd = __reservefd(start)) == -1) {
if ((newfd = __reservefd_unlocked(start)) == -1) {
_spunlock(&__fds_lock);
return -1;
}
} else {
if (__ensurefds(newfd) == -1) return -1;
if (g_fds.p[newfd].kind) close(newfd);
if (__ensurefds_unlocked(newfd) == -1) {
_spunlock(&__fds_lock);
return -1;
}
if (g_fds.p[newfd].kind) {
_spunlock(&__fds_lock);
close(newfd);
_spinlock(&__fds_lock);
}
g_fds.p[newfd].kind = kFdReserved;
}
@ -80,9 +93,12 @@ textwindows int sys_dup_nt(int oldfd, int newfd, int flags, int start) {
if (g_fds.p[oldfd].worker) {
g_fds.p[newfd].worker = weaken(RefNtStdinWorker)(g_fds.p[oldfd].worker);
}
return newfd;
rc = newfd;
} else {
__releasefd(newfd);
return __winerr();
rc = __winerr();
}
_spunlock(&__fds_lock);
return rc;
}

View file

@ -18,6 +18,6 @@
*/
#include "libc/calls/internal.h"
_Alignas(64) char __sig_lock;
_Alignas(64) int __sig_lock;
unsigned __sighandrvas[NSIG];
unsigned __sighandflags[NSIG];

View file

@ -76,8 +76,8 @@ struct Fds {
extern const struct Fd kEmptyFd;
hidden extern int __vforked;
hidden extern char __fds_lock;
hidden extern char __sig_lock;
hidden extern int __fds_lock;
hidden extern int __sig_lock;
hidden extern bool __time_critical;
hidden extern unsigned __sighandrvas[NSIG];
hidden extern unsigned __sighandflags[NSIG];
@ -85,8 +85,11 @@ hidden extern struct Fds g_fds;
hidden extern const struct NtSecurityAttributes kNtIsInheritable;
int __reservefd(int) hidden;
int __reservefd_unlocked(int) hidden;
void __releasefd(int) hidden;
void __releasefd_unlocked(int) hidden;
int __ensurefds(int) hidden;
int __ensurefds_unlocked(int) hidden;
int64_t __getfdhandleactual(int) hidden;
void __printfds(void) hidden;

View file

@ -23,11 +23,16 @@
#include "libc/calls/strace.internal.h"
#include "libc/calls/struct/sigaction.h"
#include "libc/dce.h"
#include "libc/intrin/spinlock.h"
textwindows bool _check_interrupts(bool restartable, struct Fd *fd) {
bool res;
if (__time_critical) return false;
if (_trylock(&__fds_lock)) return false;
if (weaken(_check_sigalrm)) weaken(_check_sigalrm)();
if (weaken(_check_sigchld)) weaken(_check_sigchld)();
if (fd && weaken(_check_sigwinch)) weaken(_check_sigwinch)(fd);
return weaken(__sig_check) && weaken(__sig_check)(restartable);
res = weaken(__sig_check) && weaken(__sig_check)(restartable);
_spunlock(&__fds_lock);
return res;
}

View file

@ -20,6 +20,7 @@
#include "libc/calls/internal.h"
#include "libc/calls/ntmagicpaths.internal.h"
#include "libc/calls/strace.internal.h"
#include "libc/intrin/spinlock.h"
#include "libc/nt/createfile.h"
#include "libc/nt/enum/accessmask.h"
#include "libc/nt/enum/creationdisposition.h"
@ -87,14 +88,17 @@ textwindows ssize_t sys_open_nt(int dirfd, const char *file, uint32_t flags,
int32_t mode) {
int fd;
ssize_t rc;
if ((fd = __reservefd(-1)) == -1) return -1;
if ((flags & O_ACCMODE) == O_RDWR && !strcmp(file, kNtMagicPaths.devtty)) {
rc = sys_open_nt_console(dirfd, &kNtMagicPaths, flags, mode, fd);
} else {
rc = sys_open_nt_file(dirfd, file, flags, mode, fd);
}
if (rc == -1) {
__releasefd(fd);
_spinlock(&__fds_lock);
if ((rc = fd = __reservefd_unlocked(-1)) != -1) {
if ((flags & O_ACCMODE) == O_RDWR && !strcmp(file, kNtMagicPaths.devtty)) {
rc = sys_open_nt_console(dirfd, &kNtMagicPaths, flags, mode, fd);
} else {
rc = sys_open_nt_file(dirfd, file, flags, mode, fd);
}
if (rc == -1) {
__releasefd_unlocked(fd);
}
_spunlock(&__fds_lock);
}
return rc;
}

View file

@ -17,6 +17,7 @@
PERFORMANCE OF THIS SOFTWARE.
*/
#include "libc/calls/internal.h"
#include "libc/intrin/spinlock.h"
#include "libc/nt/createfile.h"
#include "libc/nt/enum/accessmask.h"
#include "libc/nt/enum/creationdisposition.h"
@ -33,9 +34,14 @@ textwindows int sys_pipe_nt(int pipefd[2], unsigned flags) {
int reader, writer;
char16_t pipename[64];
CreatePipeName(pipename);
if ((reader = __reservefd(-1)) == -1) return -1;
if ((writer = __reservefd(-1)) == -1) {
__releasefd(reader);
_spinlock(&__fds_lock);
if ((reader = __reservefd_unlocked(-1)) == -1) {
_spunlock(&__fds_lock);
return -1;
}
if ((writer = __reservefd_unlocked(-1)) == -1) {
__releasefd_unlocked(reader);
_spunlock(&__fds_lock);
return -1;
}
if (~flags & O_DIRECT) {
@ -58,12 +64,14 @@ textwindows int sys_pipe_nt(int pipefd[2], unsigned flags) {
g_fds.p[writer].handle = hout;
pipefd[0] = reader;
pipefd[1] = writer;
_spunlock(&__fds_lock);
return 0;
} else {
CloseHandle(hin);
}
}
__releasefd(writer);
__releasefd(reader);
__releasefd_unlocked(writer);
__releasefd_unlocked(reader);
_spunlock(&__fds_lock);
return -1;
}

View file

@ -44,7 +44,7 @@
#include "libc/sysv/consts/sig.h"
#include "libc/sysv/errfuns.h"
_Alignas(64) static char poll_lock;
_Alignas(64) static int poll_lock;
/**
* Polls on the New Technology.

View file

@ -32,10 +32,9 @@
/**
* Grows file descriptor array memory if needed.
*/
int __ensurefds(int fd) {
int __ensurefds_unlocked(int fd) {
size_t n1, n2;
struct Fd *p1, *p2;
_spinlock(&__fds_lock);
n1 = g_fds.n;
if (fd >= n1) {
STRACE("__ensurefds(%d) extending", fd);
@ -48,7 +47,7 @@ int __ensurefds(int fd) {
g_fds.p = p2;
g_fds.n = n2;
if (p1 != g_fds.__init_p) {
weaken(free)(p1);
__cxa_atexit(free, p1, 0);
}
} else {
fd = enomem();
@ -57,32 +56,44 @@ int __ensurefds(int fd) {
fd = emfile();
}
}
return fd;
}
/**
* Grows file descriptor array memory if needed.
*/
int __ensurefds(int fd) {
_spinlock(&__fds_lock);
fd = __ensurefds_unlocked(fd);
_spunlock(&__fds_lock);
return fd;
}
/**
* Finds open file descriptor slot.
*/
int __reservefd_unlocked(int start) {
int fd;
for (fd = g_fds.f; fd < g_fds.n; ++fd) {
if (!g_fds.p[fd].kind) {
break;
}
}
fd = __ensurefds_unlocked(fd);
bzero(g_fds.p + fd, sizeof(*g_fds.p));
g_fds.p[fd].kind = kFdReserved;
return fd;
}
/**
* Finds open file descriptor slot.
*/
int __reservefd(int start) {
int fd;
for (;;) {
_spinlock(&__fds_lock);
fd = start < 0 ? g_fds.f : start;
while (fd < g_fds.n && g_fds.p[fd].kind) ++fd;
if (fd < g_fds.n) {
g_fds.f = fd + 1;
bzero(g_fds.p + fd, sizeof(*g_fds.p));
g_fds.p[fd].kind = kFdReserved;
_spunlock(&__fds_lock);
return fd;
} else {
_spunlock(&__fds_lock);
if (__ensurefds(fd) == -1) {
return -1;
}
}
}
_spinlock(&__fds_lock);
fd = __reservefd_unlocked(start);
_spunlock(&__fds_lock);
return fd;
}
/**
@ -91,9 +102,12 @@ int __reservefd(int start) {
static void FreeFds(void) {
int i;
NTTRACE("FreeFds()");
_spinlock(&__fds_lock);
for (i = 3; i < g_fds.n; ++i) {
if (g_fds.p[i].kind) {
_spunlock(&__fds_lock);
close(i);
_spinlock(&__fds_lock);
}
}
if (g_fds.p != g_fds.__init_p) {
@ -102,6 +116,7 @@ static void FreeFds(void) {
g_fds.p = g_fds.__init_p;
g_fds.n = ARRAYLEN(g_fds.__init_p);
}
_spunlock(&__fds_lock);
}
static textstartup void FreeFdsInit(void) {

View file

@ -223,7 +223,6 @@ static int __sigaction(int sig, const struct sigaction *act,
rc = 0;
}
if (rc != -1 && !__vforked) {
_spinlock(&__sig_lock);
if (oldact) {
oldrva = __sighandrvas[sig];
oldact->sa_sigaction = (sigaction_f)(
@ -233,7 +232,6 @@ static int __sigaction(int sig, const struct sigaction *act,
__sighandrvas[sig] = rva;
__sighandflags[sig] = act->sa_flags;
}
_spunlock(&__sig_lock);
}
return rc;
}
@ -447,7 +445,9 @@ int sigaction(int sig, const struct sigaction *act, struct sigaction *oldact) {
if (sig == SIGKILL || sig == SIGSTOP) {
rc = einval();
} else {
_spinlock(&__sig_lock);
rc = __sigaction(sig, act, oldact);
_spunlock(&__sig_lock);
}
STRACE("sigaction(%G, %s, [%s]) → %d% m", sig,
DescribeSigaction(buf[0], sizeof(buf[0]), 0, act),

View file

@ -53,7 +53,7 @@ void _check_sigchld(void) {
if (__sighandflags[SIGCHLD] & SA_NOCLDWAIT) {
STRACE("SIGCHILD SA_NOCLDWAIT fd=%d handle=%ld", pids[i], handles[i]);
CloseHandle(handles[i]);
__releasefd(pids[i]);
__releasefd_unlocked(pids[i]);
}
g_fds.p[pids[i]].zombie = true;
__sig_add(SIGCHLD, CLD_EXITED);

View file

@ -25,6 +25,7 @@
#include "libc/calls/struct/rusage.h"
#include "libc/fmt/conv.h"
#include "libc/intrin/kprintf.h"
#include "libc/intrin/spinlock.h"
#include "libc/macros.internal.h"
#include "libc/nt/accounting.h"
#include "libc/nt/enum/accessmask.h"
@ -57,6 +58,7 @@ static textwindows int sys_wait4_nt_impl(int pid, int *opt_out_wstatus,
struct NtProcessMemoryCountersEx memcount;
struct NtFileTime createfiletime, exitfiletime, kernelfiletime, userfiletime;
if (_check_interrupts(true, g_fds.p)) return eintr();
_spinlock(&__fds_lock);
if (pid != -1 && pid != 0) {
if (pid < 0) {
/* XXX: this is sloppy */
@ -67,15 +69,17 @@ static textwindows int sys_wait4_nt_impl(int pid, int *opt_out_wstatus,
if (!__isfdopen(pid) &&
(handle = OpenProcess(kNtSynchronize | kNtProcessQueryInformation,
true, pid))) {
if ((pid = __reservefd(-1)) != -1) {
if ((pid = __reservefd_unlocked(-1)) != -1) {
g_fds.p[pid].kind = kFdProcess;
g_fds.p[pid].handle = handle;
g_fds.p[pid].flags = O_CLOEXEC;
} else {
_spunlock(&__fds_lock);
CloseHandle(handle);
return echild();
}
} else {
_spunlock(&__fds_lock);
return echild();
}
}
@ -84,8 +88,12 @@ static textwindows int sys_wait4_nt_impl(int pid, int *opt_out_wstatus,
count = 1;
} else {
count = __sample_pids(pids, handles, false);
if (!count) return echild();
if (!count) {
_spunlock(&__fds_lock);
return echild();
}
}
_spunlock(&__fds_lock);
for (;;) {
if (_check_interrupts(true, 0)) return eintr();
dwExitCode = kNtStillActive;

View file

@ -27,9 +27,11 @@
#include "libc/dce.h"
#include "libc/intrin/asan.internal.h"
#include "libc/intrin/asancodes.h"
#include "libc/intrin/cmpxchg.h"
#include "libc/intrin/kprintf.h"
#include "libc/intrin/lockcmpxchg.h"
#include "libc/intrin/nomultics.internal.h"
#include "libc/intrin/spinlock.h"
#include "libc/log/backtrace.internal.h"
#include "libc/log/internal.h"
#include "libc/log/libfatal.internal.h"
@ -154,7 +156,8 @@ struct ReportOriginHeap {
int z;
};
bool __asan_noreentry;
static int __asan_noreentry;
_Alignas(64) static int __asan_lock;
static struct AsanMorgue __asan_morgue;
#define __asan_unreachable() \
@ -835,30 +838,27 @@ dontdiscard __asan_die_f *__asan_report_memory_fault(void *addr, int size,
}
void *__asan_morgue_add(void *p) {
int i;
void *r;
int i, j;
for (;;) {
i = __asan_morgue.i;
j = (i + 1) & (ARRAYLEN(__asan_morgue.p) - 1);
if (_lockcmpxchg(&__asan_morgue.i, i, j)) {
r = __asan_morgue.p[i];
__asan_morgue.p[i] = p;
return r;
}
}
_spinlock_optimistic(&__asan_lock);
i = __asan_morgue.i++ & (ARRAYLEN(__asan_morgue.p) - 1);
r = __asan_morgue.p[i];
__asan_morgue.p[i] = p;
_spunlock(&__asan_lock);
return r;
}
static void __asan_morgue_flush(void) {
int i;
void *p;
_spinlock_optimistic(&__asan_lock);
for (i = 0; i < ARRAYLEN(__asan_morgue.p); ++i) {
p = __asan_morgue.p[i];
if (_lockcmpxchg(__asan_morgue.p + i, p, 0)) {
if (weaken(dlfree)) {
weaken(dlfree)(p);
}
if (weaken(dlfree)) {
weaken(dlfree)(__asan_morgue.p[i]);
}
__asan_morgue.p[i] = 0;
}
_spunlock(&__asan_lock);
}
static size_t __asan_user_size(size_t n) {
@ -1197,12 +1197,13 @@ void __asan_evil(uint8_t *addr, int size, const char *s1, const char *s2) {
struct AsanTrace tr;
__asan_rawtrace(&tr, __builtin_frame_address(0));
kprintf(
"WARNING: ASAN error during %s bad %d byte %s at %x bt %x %x %x %x %x\n",
"WARNING: ASAN %s %s bad %d byte %s at %x bt %x %x %x %x %x\n",
__asan_noreentry == gettid() ? "error during" : "multi-threaded crash",
s1, size, s2, addr, tr.p[0], tr.p[1], tr.p[2], tr.p[3], tr.p[4], tr.p[5]);
}
void __asan_report_load(uint8_t *addr, int size) {
if (_lockcmpxchg(&__asan_noreentry, false, true)) {
if (_lockcmpxchg(&__asan_noreentry, 0, gettid())) {
if (!__vforked) {
__asan_report_memory_fault(addr, size, "load")();
__asan_unreachable();
@ -1215,7 +1216,7 @@ void __asan_report_load(uint8_t *addr, int size) {
}
void __asan_report_store(uint8_t *addr, int size) {
if (_lockcmpxchg(&__asan_noreentry, false, true)) {
if (_lockcmpxchg(&__asan_noreentry, 0, gettid())) {
if (!__vforked) {
__asan_report_memory_fault(addr, size, "store")();
__asan_unreachable();

View file

@ -16,8 +16,6 @@ struct AsanFault {
const signed char *shadow;
};
extern bool __asan_noreentry;
void __asan_unpoison(long, long);
void __asan_poison(long, long, signed char);
void __asan_verify(const void *, size_t);

View file

@ -19,6 +19,7 @@
#include "libc/assert.h"
#include "libc/bits/weaken.h"
#include "libc/calls/strace.internal.h"
#include "libc/intrin/spinlock.h"
#include "libc/macros.internal.h"
#include "libc/mem/mem.h"
#include "libc/nexgen32e/bsr.h"
@ -28,6 +29,8 @@
STATIC_YOINK("__cxa_finalize");
static int __cxa_lock;
/**
* Adds global destructor.
*
@ -47,6 +50,7 @@ noasan int __cxa_atexit(void *fp, void *arg, void *pred) {
unsigned i;
struct CxaAtexitBlock *b, *b2;
_Static_assert(ATEXIT_MAX == CHAR_BIT * sizeof(b->mask), "");
_spinlock(&__cxa_lock);
b = __cxa_blocks.p;
if (!b) b = __cxa_blocks.p = &__cxa_blocks.root;
if (!~b->mask) {
@ -55,6 +59,7 @@ noasan int __cxa_atexit(void *fp, void *arg, void *pred) {
b2->next = b;
__cxa_blocks.p = b = b2;
} else {
_spunlock(&__cxa_lock);
return enomem();
}
}
@ -64,5 +69,6 @@ noasan int __cxa_atexit(void *fp, void *arg, void *pred) {
b->p[i].fp = fp;
b->p[i].arg = arg;
b->p[i].pred = pred;
_spunlock(&__cxa_lock);
return 0;
}

View file

@ -19,7 +19,6 @@
#include "libc/calls/strace.internal.h"
#include "libc/dce.h"
#include "libc/intrin/setjmp.internal.h"
#include "libc/intrin/winthread.internal.h"
#include "libc/nt/thread.h"
#include "libc/runtime/runtime.h"
#include "libc/sysv/consts/nr.h"
@ -34,7 +33,7 @@
*/
privileged wontreturn void _Exit1(int rc) {
struct WinThread *wt;
/* STRACE("_Exit1(%d)", rc); */
STRACE("_Exit1(%d)", rc);
if (!IsWindows() && !IsMetal()) {
register long r10 asm("r10") = 0;
asm volatile("syscall"

View file

@ -25,7 +25,7 @@
STATIC_YOINK("_init_g_fds");
struct Fds g_fds;
_Alignas(64) char __fds_lock;
_Alignas(64) int __fds_lock;
textstartup void InitializeFileDescriptors(void) {
struct Fds *fds;

View file

@ -18,7 +18,7 @@
*/
#include "libc/calls/calls.h"
#include "libc/dce.h"
#include "libc/intrin/tls.h"
#include "libc/intrin/threaded.h"
#include "libc/nt/thread.h"
/**

View file

@ -31,7 +31,7 @@
#include "libc/intrin/lockcmpxchg.h"
#include "libc/intrin/nomultics.internal.h"
#include "libc/intrin/spinlock.h"
#include "libc/intrin/threaded.internal.h"
#include "libc/intrin/threaded.h"
#include "libc/limits.h"
#include "libc/log/internal.h"
#include "libc/macros.internal.h"

View file

@ -20,11 +20,15 @@
#include "libc/intrin/spinlock.h"
#include "libc/macros.internal.h"
void __releasefd(int fd) {
_spinlock(&__fds_lock);
void __releasefd_unlocked(int fd) {
if (0 <= fd && fd < g_fds.n) {
g_fds.p[fd].kind = 0;
g_fds.f = MIN(fd, g_fds.f);
}
}
void __releasefd(int fd) {
_spinlock(&__fds_lock);
__releasefd_unlocked(fd);
_spunlock(&__fds_lock);
}

View file

@ -2,16 +2,26 @@
#define COSMOPOLITAN_LIBC_INTRIN_SPINLOCK_H_
#include "libc/bits/weaken.h"
#include "libc/calls/calls.h"
#include "libc/dce.h"
#include "libc/intrin/kprintf.h"
#include "libc/intrin/lockcmpxchg.h"
#include "libc/log/backtrace.internal.h"
#include "libc/log/log.h"
#include "libc/runtime/symbols.internal.h"
#if IsModeDbg() && !defined(_SPINLOCK_DEBUG)
#define _SPINLOCK_DEBUG
#endif
#if defined(_SPINLOCK_DEBUG)
#define _spinlock(lock) _spinlock_debug(lock)
#define _spinlock(lock) _spinlock_ndebug(lock)
#define _spinlock_ndebug(lock) _spinlock_optimistic(lock)
#elif defined(TINY)
#define _spinlock(lock) _spinlock_tiny(lock)
#define _spinlock(lock) _spinlock_tiny(lock)
#define _spinlock_ndebug(lock) _spinlock_tiny(lock)
#else
#define _spinlock(lock) _spinlock_optimistic(lock)
#define _spinlock(lock) _spinlock_optimistic(lock)
#define _spinlock_ndebug(lock) _spinlock_optimistic(lock)
#endif
#define _spunlock(lock) __atomic_clear(lock, __ATOMIC_RELAXED)
@ -44,22 +54,26 @@
} \
} while (0)
#define _spinlock_debug(lock) \
do { \
typeof(*(lock)) me, owner; \
me = gettid(); \
if (_trylock(lock)) { \
__atomic_load(lock, &owner, __ATOMIC_RELAXED); \
if (owner == me) { \
kprintf("%s:%d: warning: possible spinlock re-entry in %s()\n", \
__FILE__, __LINE__, __FUNCTION__); \
if (weaken(ShowBacktrace)) { \
weaken(ShowBacktrace)(2, 0); \
} \
} \
_spinlock_optimistic(lock); \
} \
*lock = me; \
#define _spinlock_debug(lock) \
do { \
typeof(*(lock)) me, owner; \
unsigned long warntries = 10000000; \
me = gettid(); \
if (!_lockcmpxchg(lock, 0, me)) { \
__atomic_load(lock, &owner, __ATOMIC_RELAXED); \
if (owner == me) { \
kprintf("%s:%d: warning: possible re-entry on %s in %s()\n", __FILE__, \
__LINE__, #lock, __FUNCTION__); \
} \
while (!_lockcmpxchg(lock, 0, me)) { \
if (!--warntries) { \
warntries = -1; \
kprintf("%s:%d: warning: possible deadlock on %s in %s()\n", \
__FILE__, __LINE__, #lock, __FUNCTION__); \
} \
__builtin_ia32_pause(); \
} \
} \
} while (0)
#endif /* COSMOPOLITAN_LIBC_INTRIN_SPINLOCK_H_ */

16
libc/intrin/threaded.h Normal file
View file

@ -0,0 +1,16 @@
#ifndef COSMOPOLITAN_LIBC_INTRIN_THREADED_H_
#define COSMOPOLITAN_LIBC_INTRIN_THREADED_H_
#if !(__ASSEMBLER__ + __LINKER__ + 0)
COSMOPOLITAN_C_START_
extern bool __threaded;
extern bool __tls_enabled;
extern unsigned __tls_index;
void *__initialize_tls(char[hasatleast 64]);
void __install_tls(char[hasatleast 64]);
char *__get_tls(void);
COSMOPOLITAN_C_END_
#endif /* !(__ASSEMBLER__ + __LINKER__ + 0) */
#endif /* COSMOPOLITAN_LIBC_INTRIN_THREADED_H_ */

View file

@ -1,11 +0,0 @@
#ifndef COSMOPOLITAN_LIBC_INTRIN_THREADED_INTERNAL_H_
#define COSMOPOLITAN_LIBC_INTRIN_THREADED_INTERNAL_H_
#if !(__ASSEMBLER__ + __LINKER__ + 0)
COSMOPOLITAN_C_START_
extern bool __hastls;
extern bool __threaded;
COSMOPOLITAN_C_END_
#endif /* !(__ASSEMBLER__ + __LINKER__ + 0) */
#endif /* COSMOPOLITAN_LIBC_INTRIN_THREADED_INTERNAL_H_ */

View file

@ -17,76 +17,67 @@
PERFORMANCE OF THIS SOFTWARE.
*/
#include "libc/assert.h"
#include "libc/calls/calls.h"
#include "libc/dce.h"
#include "libc/intrin/tls.h"
#include "libc/errno.h"
#include "libc/intrin/threaded.h"
#include "libc/nt/thread.h"
#include "libc/nt/thunk/msabi.h"
#include "libc/sysv/consts/nrlinux.h"
__msabi extern typeof(TlsFree) *const __imp_TlsFree;
__msabi extern typeof(TlsAlloc) *const __imp_TlsAlloc;
__msabi extern typeof(TlsGetValue) *const __imp_TlsGetValue;
__msabi extern typeof(TlsSetValue) *const __imp_TlsSetValue;
#define __NR_sysarch 0x000000a5
#define __NR___set_tcb 0x00000149
#define __NR__lwp_setprivate 0x0000013d
#define __NR_thread_fast_set_cthread_self 0x03000003
/**
* Assigns thread-local storage slot.
*
* This function may for instance be called at startup and the result
* can be assigned to a global static variable; from then on, all the
* threads in your application may pass that value to TlsGetValue, to
* retrieve their thread-local values.
*
* @return index on success, or -1u w/ errno
* @threadsafe
* Initializes thread information block.
*/
uint32_t TlsAlloc(void) {
return __imp_TlsAlloc();
privileged void *__initialize_tls(char tib[hasatleast 64]) {
*(intptr_t *)tib = (intptr_t)tib;
*(intptr_t *)(tib + 0x30) = (intptr_t)tib;
*(int *)(tib + 0x3c) = __errno;
return tib;
}
/**
* Releases thread-local storage slot.
* @threadsafe
* Installs thread information block on main process.
*/
bool32 TlsFree(uint32_t dwTlsIndex) {
return __imp_TlsFree(dwTlsIndex);
}
/**
* Sets value to thread-local storage slot.
*
* @param dwTlsIndex is something returned by TlsAlloc()
* @return true if successful, otherwise false
* @threadsafe
*/
bool32 TlsSetValue(uint32_t dwTlsIndex, void *lpTlsValue) {
assert(IsWindows());
if (dwTlsIndex < 64) {
asm("mov\t%1,%%gs:%0"
: "=m"(*((long *)0x1480 + dwTlsIndex))
: "r"(lpTlsValue));
return true;
privileged void __install_tls(char tib[hasatleast 64]) {
int ax, dx;
uint64_t magic;
unsigned char *p;
if (IsWindows()) {
if (!__tls_index) {
__tls_index = TlsAlloc();
}
asm("mov\t%1,%%gs:%0" : "=m"(*((long *)0x1480 + __tls_index)) : "r"(tib));
} else if (IsFreebsd()) {
asm volatile("syscall"
: "=a"(ax)
: "0"(__NR_sysarch), "D"(129), "S"(tib)
: "rcx", "r11", "memory", "cc");
} else if (IsXnu()) {
asm volatile("syscall"
: "=a"(ax)
: "0"(__NR_thread_fast_set_cthread_self),
"D"((intptr_t)tib - 0x30)
: "rcx", "r11", "memory", "cc");
} else if (IsOpenbsd()) {
asm volatile("syscall"
: "=a"(ax)
: "0"(__NR___set_tcb), "D"(tib)
: "rcx", "r11", "memory", "cc");
} else if (IsNetbsd()) {
asm volatile("syscall"
: "=a"(ax), "=d"(dx)
: "0"(__NR__lwp_setprivate), "D"(tib)
: "rcx", "r11", "memory", "cc");
} else {
return __imp_TlsSetValue(dwTlsIndex, lpTlsValue);
}
}
/**
* Retrieves value from thread-local storage slot.
*
* @param dwTlsIndex is something returned by TlsAlloc()
* @return true if successful, otherwise false
* @threadsafe
*/
void *TlsGetValue(uint32_t dwTlsIndex) {
void *lpTlsValue;
assert(IsWindows());
if (dwTlsIndex < 64) {
asm("mov\t%%gs:%1,%0"
: "=r"(lpTlsValue)
: "m"(*((long *)0x1480 + dwTlsIndex)));
return lpTlsValue;
// // this could also be written as...
// asm("movq\t%%gs:0x30,%0" : "=a"(tib));
// return (void *)tib[0x1480 / 8 + dwTlsIndex];
} else {
return __imp_TlsGetValue(dwTlsIndex);
asm volatile("syscall"
: "=a"(ax)
: "0"(__NR_linux_arch_prctl), "D"(ARCH_SET_FS), "S"(tib)
: "rcx", "r11", "memory");
}
__tls_enabled = true;
}

View file

@ -1,13 +0,0 @@
#ifndef COSMOPOLITAN_LIBC_INTRIN_TLS_H_
#define COSMOPOLITAN_LIBC_INTRIN_TLS_H_
#if !(__ASSEMBLER__ + __LINKER__ + 0)
COSMOPOLITAN_C_START_
uint32_t TlsAlloc(void);
bool32 TlsFree(uint32_t);
bool32 TlsSetValue(uint32_t, void *);
void *TlsGetValue(uint32_t);
COSMOPOLITAN_C_END_
#endif /* !(__ASSEMBLER__ + __LINKER__ + 0) */
#endif /* COSMOPOLITAN_LIBC_INTRIN_TLS_H_ */

View file

@ -1,38 +0,0 @@
/*-*- mode:c;indent-tabs-mode:nil;c-basic-offset:2;tab-width:8;coding:utf-8 -*-│
vi: set net ft=c ts=2 sts=2 sw=2 fenc=utf-8 :vi
Copyright 2022 Justine Alexandra Roberts Tunney
Permission to use, copy, modify, and/or distribute this software for
any purpose with or without fee is hereby granted, provided that the
above copyright notice and this permission notice appear in all copies.
THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL
WARRANTIES WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED
WARRANTIES OF MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE
AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL
DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR
PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER
TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR
PERFORMANCE OF THIS SOFTWARE.
*/
#include "libc/dce.h"
#include "libc/intrin/tls.h"
#include "libc/intrin/winthread.internal.h"
/**
* @fileoverview TLS slot for clone() win32 polyfill.
*/
int __winthread;
static textstartup void __winthread_init(void) {
if (IsWindows()) {
__winthread = TlsAlloc();
TlsSetValue(__winthread, 0);
}
}
const void *const __winthread_ctor[] initarray = {
__winthread_init,
};

View file

@ -1,24 +0,0 @@
#ifndef COSMOPOLITAN_LIBC_RUNTIME_WINTHREAD_INTERNAL_H_
#define COSMOPOLITAN_LIBC_RUNTIME_WINTHREAD_INTERNAL_H_
#include "libc/intrin/tls.h"
#include "libc/runtime/runtime.h"
#if !(__ASSEMBLER__ + __LINKER__ + 0)
COSMOPOLITAN_C_START_
struct WinThread {
uint32_t tid;
int flags;
int *ctid;
int (*func)(void *);
void *arg;
};
extern int __winthread;
static inline struct WinThread *GetWinThread(void) {
return TlsGetValue(__winthread);
}
COSMOPOLITAN_C_END_
#endif /* !(__ASSEMBLER__ + __LINKER__ + 0) */
#endif /* COSMOPOLITAN_LIBC_RUNTIME_WINTHREAD_INTERNAL_H_ */

View file

@ -41,7 +41,7 @@
#define kNontrivialSize (8 * 1000 * 1000)
static struct timespec vflogf_ts;
_Alignas(64) static char vflogf_lock;
_Alignas(64) static int vflogf_lock;
/**
* Takes corrective action if logging is on the fritz.

View file

@ -1,21 +0,0 @@
/*-*- mode:c;indent-tabs-mode:nil;c-basic-offset:2;tab-width:8;coding:utf-8 -*-│
vi: set net ft=c ts=2 sts=2 sw=2 fenc=utf-8 :vi
Copyright 2022 Justine Alexandra Roberts Tunney
Permission to use, copy, modify, and/or distribute this software for
any purpose with or without fee is hereby granted, provided that the
above copyright notice and this permission notice appear in all copies.
THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL
WARRANTIES WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED
WARRANTIES OF MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE
AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL
DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR
PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER
TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR
PERFORMANCE OF THIS SOFTWARE.
*/
#include "libc/intrin/threaded.internal.h"
bool __hastls;

View file

@ -42,7 +42,6 @@ $(LIBC_NEXGEN32E_A).pkg: \
$(LIBC_NEXGEN32E_A_OBJS) \
$(foreach x,$(LIBC_NEXGEN32E_A_DIRECTDEPS),$($(x)_A).pkg)
o/$(MODE)/libc/nexgen32e/hastls.o \
o/$(MODE)/libc/nexgen32e/threaded.o: \
OVERRIDE_CFLAGS += \
$(NO_MAGIC) \

View file

@ -16,6 +16,8 @@
TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR
PERFORMANCE OF THIS SOFTWARE.
*/
#include "libc/intrin/threaded.internal.h"
#include "libc/intrin/threaded.h"
bool __threaded;
bool __tls_enabled;
unsigned __tls_index;

View file

@ -2,7 +2,7 @@
.imp kernel32,__imp_TlsAlloc,TlsAlloc,0
.text.windows
__TlsAlloc:
TlsAlloc:
push %rbp
mov %rsp,%rbp
.profilable
@ -10,5 +10,5 @@ __TlsAlloc:
call *__imp_TlsAlloc(%rip)
leave
ret
.endfn __TlsAlloc,globl
.endfn TlsAlloc,globl
.previous

View file

@ -2,7 +2,7 @@
.imp kernel32,__imp_TlsFree,TlsFree,0
.text.windows
__TlsFree:
TlsFree:
push %rbp
mov %rsp,%rbp
.profilable
@ -11,5 +11,5 @@ __TlsFree:
call *__imp_TlsFree(%rip)
leave
ret
.endfn __TlsFree,globl
.endfn TlsFree,globl
.previous

View file

@ -2,7 +2,7 @@
.imp kernel32,__imp_TlsGetValue,TlsGetValue,0
.text.windows
__TlsGetValue:
TlsGetValue:
push %rbp
mov %rsp,%rbp
.profilable
@ -11,5 +11,5 @@ __TlsGetValue:
call *__imp_TlsGetValue(%rip)
leave
ret
.endfn __TlsGetValue,globl
.endfn TlsGetValue,globl
.previous

View file

@ -2,11 +2,11 @@
.imp kernel32,__imp_TlsSetValue,TlsSetValue,0
.text.windows
__TlsSetValue:
TlsSetValue:
push %rbp
mov %rsp,%rbp
.profilable
mov __imp_TlsSetValue(%rip),%rax
jmp __sysv2nt
.endfn __TlsSetValue,globl
.endfn TlsSetValue,globl
.previous

View file

@ -302,8 +302,8 @@ imp 'EnumerateLocalComputerNamesA' EnumerateLocalComputerNamesA kernel32 3
imp 'EraseTape' EraseTape kernel32 352
imp 'EscapeCommFunction' EscapeCommFunction kernel32 0
imp 'ExecuteUmsThread' ExecuteUmsThread kernel32 354
imp 'ExitThread' ExitThread kernel32 0 1
imp 'ExitProcess' ExitProcess kernel32 0 1 # a.k.a. RtlExitUserProcess
imp 'ExitThread' ExitThread kernel32 0 1
imp 'ExitVDM' ExitVDM kernel32 357
imp 'ExpandEnvironmentStrings' ExpandEnvironmentStringsW kernel32 0
imp 'ExpandEnvironmentStringsA' ExpandEnvironmentStringsA kernel32 0
@ -1225,6 +1225,10 @@ imp 'TermsrvSetValueKey' TermsrvSetValueKey kernel32 1441
imp 'TermsrvSyncUserIniFileExt' TermsrvSyncUserIniFileExt kernel32 1442
imp 'Thread32First' Thread32First kernel32 1443
imp 'Thread32Next' Thread32Next kernel32 1444
imp 'TlsAlloc' TlsAlloc kernel32 0 0
imp 'TlsFree' TlsFree kernel32 0 1
imp 'TlsGetValue' TlsGetValue kernel32 0 1
imp 'TlsSetValue' TlsSetValue kernel32 0 2
imp 'Toolhelp32ReadProcessMemory' Toolhelp32ReadProcessMemory kernel32 1449
imp 'TransactNamedPipe' TransactNamedPipe kernel32 0 7
imp 'TransmitCommChar' TransmitCommChar kernel32 0
@ -1364,10 +1368,6 @@ imp '__ReOpenFile' ReOpenFile kernel32 0 4 # TODO(jart): 6.2 and highe
imp '__RemoveDirectory' RemoveDirectoryW kernel32 0 1
imp '__SetCurrentDirectory' SetCurrentDirectoryW kernel32 0 1
imp '__TerminateProcess' TerminateProcess kernel32 0 2
imp '__TlsAlloc' TlsAlloc kernel32 0 0
imp '__TlsFree' TlsFree kernel32 0 1
imp '__TlsGetValue' TlsGetValue kernel32 0 1
imp '__TlsSetValue' TlsSetValue kernel32 0 2
imp '__UnmapViewOfFile' UnmapViewOfFile kernel32 0 1
imp '__VirtualProtect' VirtualProtect kernel32 0 4
imp '__WaitForMultipleObjects' WaitForMultipleObjects kernel32 0 4

View file

@ -57,6 +57,11 @@ bool32 CancelSynchronousIo(int64_t hThread);
bool32 CancelIo(int64_t hFile);
bool32 CancelIoEx(int64_t hFile, struct NtOverlapped *opt_lpOverlapped);
uint32_t TlsAlloc(void);
bool32 TlsFree(uint32_t);
bool32 TlsSetValue(uint32_t, void *);
void *TlsGetValue(uint32_t);
#if ShouldUseMsabiAttribute()
#include "libc/nt/thunk/thread.inc"
#endif /* ShouldUseMsabiAttribute() */

View file

@ -26,9 +26,7 @@
#include "libc/intrin/asan.internal.h"
#include "libc/intrin/kprintf.h"
#include "libc/intrin/spinlock.h"
#include "libc/intrin/threaded.internal.h"
#include "libc/intrin/tls.h"
#include "libc/intrin/winthread.internal.h"
#include "libc/intrin/threaded.h"
#include "libc/nt/runtime.h"
#include "libc/nt/thread.h"
#include "libc/nt/thunk/msabi.h"
@ -46,58 +44,22 @@ STATIC_YOINK("gettid"); // for kprintf()
#define __NR_clone_linux 56
#define __NR__lwp_create 309
#define __NR_getcontext_netbsd 307
#define __NR__lwp_setprivate 317
#define __NR_bsdthread_create 0x02000168
#define __NR_thread_fast_set_cthread_self 0x03000003
#define __NR_sysarch 0x000000a5
#define __NR___set_tcb 0x00000149
#define PTHREAD_START_CUSTOM_XNU 0x01000000
#define LWP_DETACHED 0x00000040
#define LWP_SUSPENDED 0x00000080
char __tls[512];
int __errno_global;
extern int __errno_index;
static char tibdefault[64];
privileged void __setup_tls(void) {
int ax, dx;
uint64_t magic;
unsigned char *p;
*(intptr_t *)__tls = (intptr_t)__tls;
*(intptr_t *)(__tls + 0x30) = (intptr_t)__tls;
*(int *)(__tls + 0x3c) = __errno;
if (IsWindows()) {
__errno_index = TlsAlloc();
TlsSetValue(__errno_index, (void *)(intptr_t)__errno);
} else if (IsLinux()) {
asm volatile("syscall"
: "=a"(ax)
: "0"(__NR_linux_arch_prctl), "D"(ARCH_SET_FS), "S"(__tls)
: "rcx", "r11", "memory");
} else if (IsFreebsd()) {
asm volatile("syscall"
: "=a"(ax)
: "0"(__NR_sysarch), "D"(129), "S"(__tls)
: "rcx", "r11", "memory", "cc");
} else if (IsXnu()) {
asm volatile("syscall"
: "=a"(ax)
: "0"(__NR_thread_fast_set_cthread_self),
"D"((intptr_t)__tls - 0x30)
: "rcx", "r11", "memory", "cc");
} else if (IsOpenbsd()) {
asm volatile("syscall"
: "=a"(ax)
: "0"(__NR___set_tcb), "D"(__tls)
: "rcx", "r11", "memory", "cc");
} else if (IsNetbsd()) {
asm volatile("syscall"
: "=a"(ax), "=d"(dx)
: "0"(__NR__lwp_setprivate), "D"(__tls)
: "rcx", "r11", "memory", "cc");
}
__hastls = true;
}
struct WinThread {
uint32_t tid;
int flags;
int *ctid;
void *tls;
int (*func)(void *);
void *arg;
};
uint32_t WinThreadThunk(void *warg);
asm(".section\t.text.windows,\"ax\",@progbits\n\t"
@ -115,20 +77,19 @@ __attribute__((__used__, __no_reorder__))
static textwindows wontreturn void
WinThreadMain(struct WinThread *wt) {
int rc;
if (wt->flags & CLONE_SETTLS) {
TlsSetValue(__tls_index, wt->tls);
}
if (wt->flags & CLONE_CHILD_SETTID) {
*wt->ctid = wt->tid;
}
rc = wt->func(wt->arg);
if (wt->flags & CLONE_CHILD_CLEARTID) {
*wt->ctid = 0;
}
_Exit1(rc);
}
static textwindows int CloneWindows(int (*func)(void *), char *stk,
size_t stksz, int flags, void *arg,
int *ptid, void *tls, size_t tlssz,
int *ctid) {
void *tls, size_t tlssz, int *ctid) {
int64_t h;
struct WinThread *wt;
wt = (struct WinThread *)(((intptr_t)(stk + stksz) -
@ -138,14 +99,11 @@ static textwindows int CloneWindows(int (*func)(void *), char *stk,
wt->ctid = ctid;
wt->func = func;
wt->arg = arg;
wt->tls = tls;
if ((h = CreateThread(0, 0, WinThreadThunk, wt, 0, &wt->tid))) {
CloseHandle(h);
if (flags & CLONE_PARENT_SETTID) {
*ptid = wt->tid;
}
return wt->tid;
} else {
__releasefd(wt->tid);
return -1;
}
}
@ -179,14 +137,11 @@ XnuThreadMain(void *pthread, int tid, int (*func)(void *arg), void *arg,
*(int *)sp[2] = tid;
}
rc = func(arg);
if (sp[4] & CLONE_CHILD_CLEARTID) {
*(int *)sp[2] = 0;
}
_Exit1(rc);
}
static int CloneXnu(int (*fn)(void *), char *stk, size_t stksz, int flags,
void *arg, int *ptid, void *tls, size_t tlssz, int *ctid) {
void *arg, void *tls, size_t tlssz, int *ctid) {
int rc;
bool failed;
intptr_t *sp;
@ -212,9 +167,6 @@ static int CloneXnu(int (*fn)(void *), char *stk, size_t stksz, int flags,
_seizelock(sp); // TODO: How can we get the tid without locking?
if ((rc = bsdthread_create(fn, arg, sp, 0, PTHREAD_START_CUSTOM_XNU)) != -1) {
_spinlock(sp);
if (flags & CLONE_PARENT_SETTID) {
*ptid = sp[1];
}
rc = sp[1];
}
return rc;
@ -236,15 +188,11 @@ FreebsdThreadMain(intptr_t *sp) {
*(int *)sp[2] = sp[4];
}
rc = ((int (*)(intptr_t))sp[0])(sp[1]);
if (sp[3] & CLONE_CHILD_CLEARTID) {
*(int *)sp[2] = 0;
}
_Exit1(rc);
}
static int CloneFreebsd(int (*func)(void *), char *stk, size_t stksz, int flags,
void *arg, int *ptid, void *tls, size_t tlssz,
int *ctid) {
void *arg, void *tls, size_t tlssz, int *ctid) {
int ax;
bool failed;
int64_t tid;
@ -270,15 +218,11 @@ static int CloneFreebsd(int (*func)(void *), char *stk, size_t stksz, int flags,
: CFLAG_CONSTRAINT(failed), "=a"(ax)
: "1"(__NR_thr_new), "D"(&params), "S"(sizeof(params))
: "rcx", "rdx", "r8", "r9", "r10", "r11", "memory");
if (!failed) {
if (flags & CLONE_PARENT_SETTID) {
*ptid = tid;
}
return tid;
} else {
if (failed) {
errno = ax;
return -1;
tid = -1;
}
return tid;
}
struct __tfork {
@ -313,15 +257,11 @@ static privileged wontreturn void
OpenbsdThreadMain(intptr_t *sp) {
int rc;
rc = ((int (*)(intptr_t))sp[0])(sp[1]);
if (sp[3] & CLONE_CHILD_CLEARTID) {
*(int *)sp[2] = 0;
}
_Exit1(rc);
}
static int CloneOpenbsd(int (*func)(void *), char *stk, size_t stksz, int flags,
void *arg, int *ptid, void *tls, size_t tlssz,
int *ctid) {
void *arg, void *tls, size_t tlssz, int *ctid) {
int tid;
intptr_t *sp;
struct __tfork params;
@ -333,11 +273,7 @@ static int CloneOpenbsd(int (*func)(void *), char *stk, size_t stksz, int flags,
params.tf_stack = sp;
params.tf_tcb = flags & CLONE_SETTLS ? tls : 0;
params.tf_tid = flags & CLONE_CHILD_SETTID ? ctid : 0;
if ((tid = __tfork(&params, sizeof(params), sp)) > 0) {
if (flags & CLONE_PARENT_SETTID) {
*ptid = tid;
}
} else {
if ((tid = __tfork(&params, sizeof(params), sp)) < 0) {
errno = -tid;
tid = -1;
}
@ -351,15 +287,11 @@ static wontreturn void NetbsdThreadMain(void *arg, int (*func)(void *arg),
*ctid = *tid;
}
rc = func(arg);
if (flags & CLONE_CHILD_CLEARTID) {
*ctid = 0;
}
_Exit1(rc);
}
static int CloneNetbsd(int (*func)(void *), char *stk, size_t stksz, int flags,
void *arg, int *ptid, void *tls, size_t tlssz,
int *ctid) {
void *arg, void *tls, size_t tlssz, int *ctid) {
// NetBSD has its own clone() and it works, but it's technically a
// second-class API, intended to help Linux folks migrate to this!
// We put it on the thread's stack, to avoid locking this function
@ -414,9 +346,6 @@ static int CloneNetbsd(int (*func)(void *), char *stk, size_t stksz, int flags,
: "1"(__NR__lwp_create), "D"(ctx), "S"(LWP_DETACHED), "2"(tid)
: "rcx", "r11", "memory");
if (!failed) {
if (flags & CLONE_PARENT_SETTID) {
*ptid = *tid;
}
return *tid;
} else {
errno = ax;
@ -453,49 +382,65 @@ static int CloneLinux(int (*func)(void *), char *stk, size_t stksz, int flags,
* Creates thread.
*
* Threads are created in a detached manner. They currently can't be
* synchronized using wait() and posix signals. Threads created by this
* synchronized using wait() or posix signals. Threads created by this
* function should be synchronized using shared memory operations.
*
* Any memory that's required by this system call wrapper is allocated
* to the top of your stack. This is normally about 64 bytes, although
* on NetBSD it's currently 800.
*
* Your function is called from within the stack you specify. A return
* address is pushed onto your stack, that causes returning to jump to
* _Exit1() which terminates the thread. Even though the callback says
* it supports a return code, that'll only work on Linux and Windows.
*
* The `tls` parameter is for thread-local storage. If you specify this
* then clone() will implicitly rewire libc (e.g. errno) to use TLS:
*
* static char tib[64];
* __initialize_tls(tib);
* __install_tls(tib);
*
* If you want a main process TLS size that's larger call it manually.
* Once you've done the above and/or started creating your own threads
* you'll be able to access your `tls` thread information block, using
*
* char *p = __get_tls();
* printf("errno is %d\n", *(int *)(p + 0x3c));
*
* This function follows the same ABI convention as the Linux userspace
* libraries, with a few small changes. The varargs has been removed to
* help prevent broken code, and the stack size and tls size parameters
* are introduced for compatibility with FreeBSD.
*
* To keep this system call lightweight, only the thread creation use
* case is polyfilled across platforms. For example, if you want fork
* that works on OpenBSD for example, don't do it with clone(SIGCHLD)
* and please just call fork(). Even if you do that on Linux, it will
* effectively work around libc features like atfork(), so that means
* other calls like getpid() may return incorrect values.
*
* @param func is your callback function
* @param stk points to the bottom of a caller allocated stack, which
* must be null when fork() and vfork() equivalent flags are used
* and furthermore this must be mmap()'d using MAP_STACK in order
* to work on OpenBSD
* @param stksz is the size of that stack in bytes which must be zero
* if the fork() or vfork() equivalent flags are used it's highly
* recommended that this value be GetStackSize(), or else kprintf
* and other runtime services providing memory safety can't do as
* good and quick of a job; this value must be 16-aligned plus it
* must be at minimum 4096 bytes in size
* @param flags usually has one of
* - `SIGCHLD` will delegate to fork()
* - `CLONE_VFORK|CLONE_VM|SIGCHLD` means vfork()
* must be allocated via mmap() using the MAP_STACK flag, or else
* you won't get optimal performance and it won't work on OpenBSD
* @param stksz is the size of that stack in bytes, we recommend that
* that this be set to GetStackSize() or else memory safety tools
* like kprintf() can't do as good and quick of a job; this value
* must be 16-aligned plus it must be at least 4192 bytes in size
* and it's advised to have the bottom-most page, be a guard page
* @param flags should have:
* - `CLONE_THREAD|CLONE_VM|CLONE_FS|CLONE_FILES|CLONE_SIGHAND`
* as part high bytes, and the low order byte may optionally contain
* a signal e.g. SIGCHLD, to enable parent notification on terminate
* although the signal isn't supported on non-Linux and non-NetBSD
* at the moment; 'flags' may optionally bitwise or the following:
* - `CLONE_PARENT_SETTID` is needed for `ctid` should be set
* - `CLONE_CHILD_SETTID` is needed for `ptid` should be set
* - `CLONE_SETTLS` is needed to set `%fs` segment to `tls`
* and may optionally bitwise any of the following:
* - `CLONE_CHILD_SETTID` is needed too if you use `ctid`
* - `CLONE_SETTLS` is needed too if you set `tls`
* @param arg will be passed to your callback
* @param ptid lets the parent receive the child thread id;
* this parameter is ignored if `CLONE_PARENT_SETTID` is not set
* @param tls may be used to set the thread local storage segment;
* this parameter is ignored if `CLONE_SETTLS` is not set
* @param tlssz is the size of tls in bytes
* @param ctid lets the child receive its thread id;
* this parameter is ignored if `CLONE_CHILD_SETTID` is not set
* @return tid on success and 0 to the child, or -1 w/ errno
* @param tlssz is the size of tls in bytes which must be at least 64
* @param ctid lets the child receive its thread id without having to
* call gettid() and is ignored if `CLONE_CHILD_SETTID` isn't set
* @return tid of child on success, or -1 w/ errno
* @threadsafe
*/
int clone(int (*func)(void *), void *stk, size_t stksz, int flags, void *arg,
@ -503,11 +448,11 @@ int clone(int (*func)(void *), void *stk, size_t stksz, int flags, void *arg,
int rc;
__threaded = true;
if (tls && !__hastls) {
__setup_tls();
if (tls && !__tls_enabled) {
__initialize_tls(tibdefault);
__install_tls(tibdefault);
}
// verify memory is kosher
if (IsAsan() &&
((stksz > PAGESIZE &&
!__asan_is_valid((char *)stk + PAGESIZE, stksz - PAGESIZE)) ||
@ -518,54 +463,27 @@ int clone(int (*func)(void *), void *stk, size_t stksz, int flags, void *arg,
((flags & CLONE_CHILD_SETTID) &&
!__asan_is_valid(ctid, sizeof(*ctid))))) {
rc = efault();
}
// delegate to bona fide clone()
else if (IsLinux()) {
} else if (!IsTiny() &&
(((flags & CLONE_VM) && (stksz < PAGESIZE || (stksz & 15))) ||
((flags & CLONE_SETTLS) && (tlssz < 64 || (tlssz & 7))))) {
rc = einval();
} else if (IsLinux()) {
rc = CloneLinux(func, stk, stksz, flags, arg, ptid, tls, tlssz, ctid);
}
// polyfill fork() and vfork() use cases on platforms without clone()
else if ((SupportsWindows() || SupportsBsd()) &&
flags == (CLONE_VFORK | CLONE_VM | SIGCHLD)) {
if (IsTiny()) {
rc = einval();
} else if (!arg && !stksz) {
return vfork(); // don't log clone()
} else {
rc = einval();
}
} else if ((SupportsWindows() || SupportsBsd()) && flags == SIGCHLD) {
if (IsTiny()) {
rc = eopnotsupp();
} else if (!arg && !stksz) {
return fork(); // don't log clone()
} else {
rc = einval();
}
}
// we now assume we're creating a thread
// these platforms can't do signals the way linux does
else if (!IsTiny() && ((stksz < PAGESIZE || (stksz & 15)) ||
(flags & ~(CLONE_SETTLS | CLONE_PARENT_SETTID |
CLONE_CHILD_SETTID)) !=
(CLONE_THREAD | CLONE_VM | CLONE_FS | CLONE_FILES |
CLONE_SIGHAND))) {
} else if (!IsTiny() && (flags & ~(CLONE_SETTLS | CLONE_PARENT_SETTID |
CLONE_CHILD_SETTID)) !=
(CLONE_THREAD | CLONE_VM | CLONE_FS |
CLONE_FILES | CLONE_SIGHAND)) {
rc = einval();
} else if (IsXnu()) {
rc = CloneXnu(func, stk, stksz, flags, arg, ptid, tls, tlssz, ctid);
rc = CloneXnu(func, stk, stksz, flags, arg, tls, tlssz, ctid);
} else if (IsFreebsd()) {
rc = CloneFreebsd(func, stk, stksz, flags, arg, ptid, tls, tlssz, ctid);
rc = CloneFreebsd(func, stk, stksz, flags, arg, tls, tlssz, ctid);
} else if (IsNetbsd()) {
rc = CloneNetbsd(func, stk, stksz, flags, arg, ptid, tls, tlssz, ctid);
rc = CloneNetbsd(func, stk, stksz, flags, arg, tls, tlssz, ctid);
} else if (IsOpenbsd()) {
rc = CloneOpenbsd(func, stk, stksz, flags, arg, ptid, tls, tlssz, ctid);
}
// These platforms can't do segment registers like linux does
else if (IsWindows()) {
rc = CloneWindows(func, stk, stksz, flags, arg, ptid, tls, tlssz, ctid);
rc = CloneOpenbsd(func, stk, stksz, flags, arg, tls, tlssz, ctid);
} else if (IsWindows()) {
rc = CloneWindows(func, stk, stksz, flags, arg, tls, tlssz, ctid);
} else {
rc = enosys();
}

View file

@ -20,6 +20,7 @@
#include "libc/bits/bits.h"
#include "libc/bits/weaken.h"
#include "libc/calls/strace.internal.h"
#include "libc/intrin/spinlock.h"
#include "libc/macros.internal.h"
#include "libc/runtime/runtime.h"
#include "libc/runtime/symbols.internal.h"
@ -29,6 +30,7 @@
#include "libc/zip.h"
#include "libc/zipos/zipos.internal.h"
static char g_lock;
static struct SymbolTable *g_symtab;
/**
@ -118,6 +120,7 @@ static struct SymbolTable *GetSymbolTableFromElf(void) {
*/
struct SymbolTable *GetSymbolTable(void) {
struct Zipos *z;
if (_trylock(&g_lock)) return 0;
if (!g_symtab && !__isworker) {
if (weaken(__zipos_get) && (z = weaken(__zipos_get)())) {
if ((g_symtab = GetSymbolTableFromZip(z))) {
@ -131,6 +134,7 @@ struct SymbolTable *GetSymbolTable(void) {
g_symtab = GetSymbolTableFromElf();
}
}
_spunlock(&g_lock);
return g_symtab;
}

View file

@ -46,7 +46,7 @@ struct MemoryIntervals {
size_t i, n;
struct MemoryInterval *p;
struct MemoryInterval s[OPEN_MAX];
_Alignas(64) char lock;
_Alignas(64) int lock;
};
extern hidden struct MemoryIntervals _mmi;

View file

@ -16,6 +16,9 @@
TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR
PERFORMANCE OF THIS SOFTWARE.
*/
#include "libc/bits/weaken.h"
#include "libc/intrin/kprintf.h"
#include "libc/log/backtrace.internal.h"
#include "libc/mem/mem.h"
#include "libc/sock/internal.h"

View file

@ -20,6 +20,7 @@
#include "libc/calls/calls.h"
#include "libc/calls/strace.internal.h"
#include "libc/dce.h"
#include "libc/intrin/spinlock.h"
#include "libc/mem/mem.h"
#include "libc/nt/runtime.h"
#include "libc/nt/winsock.h"
@ -40,12 +41,6 @@ hidden struct NtWsaData kNtWsaData;
static textwindows void WinSockCleanup(void) {
int i, rc;
NTTRACE("WinSockCleanup()");
for (i = g_fds.n; i--;) {
if (g_fds.p[i].kind == kFdSocket) {
close(i);
}
}
// TODO(jart): Check WSACleanup() result code
rc = WSACleanup();
NTTRACE("WSACleanup() → %d% lm", rc);
}

View file

@ -17,6 +17,7 @@
PERFORMANCE OF THIS SOFTWARE.
*/
#include "libc/calls/internal.h"
#include "libc/intrin/spinlock.h"
#include "libc/mem/mem.h"
#include "libc/nt/enum/fileflagandattributes.h"
#include "libc/nt/iphlpapi.h"
@ -61,11 +62,13 @@ textwindows int sys_socket_nt(int family, int type, int protocol) {
sockfd->family = family;
sockfd->type = truetype;
sockfd->protocol = protocol;
_spinlock(&__fds_lock);
g_fds.p[fd].kind = kFdSocket;
g_fds.p[fd].flags = oflags;
g_fds.p[fd].mode = 0140666;
g_fds.p[fd].handle = h;
g_fds.p[fd].extra = (uintptr_t)sockfd;
_spunlock(&__fds_lock);
return fd;
} else {
__releasefd(fd);

View file

@ -16,6 +16,7 @@
TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR
PERFORMANCE OF THIS SOFTWARE.
*/
#include "libc/intrin/spinlock.h"
#include "libc/nt/createfile.h"
#include "libc/nt/enum/accessmask.h"
#include "libc/nt/enum/creationdisposition.h"
@ -74,6 +75,8 @@ textwindows int sys_socketpair_nt(int family, int type, int proto, int sv[2]) {
return -1;
}
_spinlock(&__fds_lock);
g_fds.p[reader].kind = kFdFile;
g_fds.p[reader].flags = oflags;
g_fds.p[reader].mode = 0140444;
@ -84,6 +87,8 @@ textwindows int sys_socketpair_nt(int family, int type, int proto, int sv[2]) {
g_fds.p[writer].mode = 0140222;
g_fds.p[writer].handle = h1;
_spunlock(&__fds_lock);
sv[0] = reader;
sv[1] = writer;
return 0;

View file

@ -10,7 +10,7 @@ struct StdioFlushHandles {
};
struct StdioFlush {
char lock;
int lock;
struct StdioFlushHandles handles;
FILE *handles_initmem[8];
};

View file

@ -24,7 +24,7 @@ typedef struct FILE {
uint32_t size; /* 0x20 */
uint32_t nofree; /* 0x24 */
int pid; /* 0x28 */
char lock; /* 0x2c */
int lock; /* 0x2c */
char *getln; /* 0x30 */
} FILE;

View file

@ -19,7 +19,8 @@
#include "libc/bits/weaken.h"
#include "libc/calls/calls.h"
#include "libc/dce.h"
#include "libc/intrin/threaded.internal.h"
#include "libc/intrin/threaded.h"
#include "libc/nt/thread.h"
/**
* Global variable for last error.
@ -34,22 +35,29 @@
* @see __errno_location() stable abi
*/
errno_t __errno;
int __errno_index;
privileged nocallersavedregisters errno_t *(__errno_location)(void) {
char *tib;
if (!__hastls) {
return &__errno;
} else if (IsLinux() || IsFreebsd() || IsNetbsd() || IsOpenbsd()) {
asm("mov\t%%fs:0,%0" : "=a"(tib));
return (errno_t *)(tib + 0x3c);
} else if (IsXnu()) {
asm("mov\t%%gs:0x30,%0" : "=a"(tib));
return (errno_t *)(tib + 0x3c);
} else if (IsWindows()) {
asm("mov\t%%gs:0x30,%0" : "=a"(tib));
return (errno_t *)(tib + 0x1480 + __errno_index * 8);
/**
* Returns address of thread information block.
* @see __install_tls()
* @see clone()
*/
privileged nocallersavedregisters char *__get_tls(void) {
char *tib, *linear = (char *)0x30;
if (IsLinux() || IsFreebsd() || IsNetbsd() || IsOpenbsd()) {
asm("mov\t%%fs:(%1),%0" : "=a"(tib) : "r"(linear));
} else {
return &__errno;
asm("mov\t%%gs:(%1),%0" : "=a"(tib) : "r"(linear));
if (IsWindows()) tib = *(char **)(tib + 0x1480 + __tls_index * 8);
}
return tib;
}
/**
* Returns address of errno variable.
* @see __initialize_tls()
* @see __install_tls()
*/
privileged nocallersavedregisters errno_t *(__errno_location)(void) {
if (!__tls_enabled) return &__errno;
return (errno_t *)(__get_tls() + 0x3c);
}

View file

@ -153,6 +153,7 @@ noasan int main(int argc, char *argv[]) {
__log_level = kLogInfo;
GetOpts(argc, argv);
setenv("GDB", "", true);
GetSymbolTable();
// normalize this process
FixIrregularFds();

View file

@ -42,7 +42,7 @@ STATIC_YOINK("usr/share/zoneinfo/UTC");
** POSIX-style TZ environment variable handling from Guy Harris.
*/
_Alignas(64) static char locallock;
_Alignas(64) static int locallock;
static int lock(void) {
_spinlock(&locallock);