mirrors/cosmopolitan
1
0
Fork 0
mirror of https://github.com/jart/cosmopolitan.git synced 2025-01-31 03:27:39 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

Fix many thread and file descriptor issues on Windows

Browse source 
The greenbean web server now works nearly perfectly on Windows with over
1000 threads. But some synchronization issues still remain which prevent
us from going over nine thousand.
This commit is contained in:
Justine Tunney 2022-05-20 18:51:41 -07:00
parent 96781d0679
commit 7838edae88
32 changed files with 363 additions and 192 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

5
examples/greenbean.c
View file

@ -100,7 +100,7 @@ int workers;
int messages;
int connections;
const char *status;
volatile int closingtime;
_Atomic(int) closingtime;
int Worker(void *id) {
int server, yes = 1;
@ -273,8 +273,7 @@ int main(int argc, char *argv[]) {
MAP_STACK | MAP_ANONYMOUS, -1, 0);
CHECK_NE(-1, clone(Worker, stack, GetStackSize(),
CLONE_THREAD | CLONE_VM | CLONE_FS | CLONE_FILES |
CLONE_SIGHAND | CLONE_SETTLS | CLONE_CHILD_SETTID |
CLONE_CHILD_CLEARTID,
CLONE_SIGHAND | CLONE_SETTLS,
(void *)(intptr_t)i, 0, tls, 64, 0));
}
status = "";

39
libc/calls/dup-nt.c
View file

@ -48,26 +48,33 @@ textwindows int sys_dup_nt(int oldfd, int newfd, int flags, int start) {
}
// allocate a new file descriptor
if (newfd == -1) {
if ((newfd = __reservefd_unlocked(start)) == -1) {
_spunlock(&__fds_lock);
return -1;
for (;;) {
if (newfd == -1) {
if ((newfd = __reservefd_unlocked(start)) == -1) {
_spunlock(&__fds_lock);
return -1;
}
break;
} else {
if (__ensurefds_unlocked(newfd) == -1) {
_spunlock(&__fds_lock);
return -1;
}
if (g_fds.p[newfd].kind) {
_spunlock(&__fds_lock);
close(newfd);
_spinlock(&__fds_lock);
}
if (!g_fds.p[newfd].kind) {
g_fds.p[newfd].kind = kFdReserved;
break;
}
}
} else {
if (__ensurefds_unlocked(newfd) == -1) {
_spunlock(&__fds_lock);
return -1;
}
if (g_fds.p[newfd].kind) {
_spunlock(&__fds_lock);
close(newfd);
_spinlock(&__fds_lock);
}
g_fds.p[newfd].kind = kFdReserved;
}
handle = g_fds.p[oldfd].handle;
proc = GetCurrentProcess();
if (DuplicateHandle(proc, handle, proc, &g_fds.p[newfd].handle, 0, true,
kNtDuplicateSameAccess)) {
g_fds.p[newfd].kind = g_fds.p[oldfd].kind;
@ -82,7 +89,7 @@ textwindows int sys_dup_nt(int oldfd, int newfd, int flags, int start) {
}
rc = newfd;
} else {
__releasefd(newfd);
__releasefd_unlocked(newfd);
rc = __winerr();
}

2
libc/calls/internal.h
View file

@ -66,7 +66,7 @@ struct Fd {
};
struct Fds {
size_t f; /* lowest free slot */
int f; /* lowest free slot */
size_t n; /* monotonic capacity */
struct Fd *p;
struct Fd __init_p[OPEN_MAX];

25
libc/calls/interrupts-nt.c
View file

@ -23,16 +23,35 @@
#include "libc/calls/strace.internal.h"
#include "libc/calls/struct/sigaction.h"
#include "libc/dce.h"
#include "libc/intrin/spinlock.h"
#include "libc/intrin/lockcmpxchgp.h"
#include "libc/nexgen32e/threaded.h"
_Alignas(64) static int rlock;
static privileged inline bool AcquireInterruptPollLock(void) {
// any thread can poll for interrupts
// but it's wasteful to have every single thread doing it
int me, owner, tries;
if (!__threaded) return true;
me = gettid();
owner = 0;
if (_lockcmpxchgp(&rlock, &owner, me)) return true;
return owner == me;
}
static textwindows inline void ReleaseInterruptPollLock(void) {
int zero = 0;
__atomic_store(&rlock, &zero, __ATOMIC_RELAXED);
}
textwindows bool _check_interrupts(bool restartable, struct Fd *fd) {
bool res;
if (__time_critical) return false;
if (_trylock(&__fds_lock)) return false;
if (!AcquireInterruptPollLock()) return false;
if (weaken(_check_sigalrm)) weaken(_check_sigalrm)();
if (weaken(_check_sigchld)) weaken(_check_sigchld)();
if (fd && weaken(_check_sigwinch)) weaken(_check_sigwinch)(fd);
res = weaken(__sig_check) && weaken(__sig_check)(restartable);
_spunlock(&__fds_lock);
ReleaseInterruptPollLock();
return res;
}

14
libc/calls/ioctl_tiocgwinsz-nt.c
View file

@ -22,6 +22,7 @@
#include "libc/calls/strace.internal.h"
#include "libc/calls/struct/termios.h"
#include "libc/calls/struct/winsize.h"
#include "libc/intrin/spinlock.h"
#include "libc/log/log.h"
#include "libc/nt/console.h"
#include "libc/nt/enum/startf.h"
@ -31,13 +32,15 @@
#include "libc/sysv/errfuns.h"
textwindows int ioctl_tiocgwinsz_nt(struct Fd *fd, struct winsize *ws) {
int i, e;
int i, e, rc;
uint32_t mode;
struct Fd *fds[3];
struct NtStartupInfo startinfo;
struct NtConsoleScreenBufferInfoEx sbinfo;
rc = -1;
e = errno;
if (ws) {
_spinlock(&__fds_lock);
fds[0] = fd, fds[1] = g_fds.p + 1, fds[2] = g_fds.p + 0;
GetStartupInfo(&startinfo);
for (i = 0; i < ARRAYLEN(fds); ++i) {
@ -51,14 +54,16 @@ textwindows int ioctl_tiocgwinsz_nt(struct Fd *fd, struct winsize *ws) {
ws->ws_xpixel = 0;
ws->ws_ypixel = 0;
errno = e;
return 0;
rc = 0;
break;
} else if (startinfo.dwFlags & kNtStartfUsecountchars) {
ws->ws_col = startinfo.dwXCountChars;
ws->ws_row = startinfo.dwYCountChars;
ws->ws_xpixel = 0;
ws->ws_ypixel = 0;
errno = e;
return 0;
rc = 0;
break;
} else {
__winerr();
}
@ -69,8 +74,9 @@ textwindows int ioctl_tiocgwinsz_nt(struct Fd *fd, struct winsize *ws) {
ebadf();
}
}
_spunlock(&__fds_lock);
} else {
efault();
}
return -1;
return rc;
}

8
libc/calls/pipe-nt.c
View file

@ -49,9 +49,11 @@ textwindows int sys_pipe_nt(int pipefd[2], unsigned flags) {
} else {
mode = kNtPipeTypeMessage | kNtPipeReadmodeMessage;
}
if ((hin = CreateNamedPipe(
pipename, kNtPipeAccessInbound | kNtFileFlagOverlapped, mode, 1,
PIPE_BUF, PIPE_BUF, 0, &kNtIsInheritable)) != -1) {
_spunlock(&__fds_lock);
hin = CreateNamedPipe(pipename, kNtPipeAccessInbound | kNtFileFlagOverlapped,
mode, 1, PIPE_BUF, PIPE_BUF, 0, &kNtIsInheritable);
_spinlock(&__fds_lock);
if (hin != -1) {
if ((hout = CreateFile(pipename, kNtGenericWrite, 0, &kNtIsInheritable,
kNtOpenExisting, kNtFileFlagOverlapped, 0)) != -1) {
g_fds.p[reader].kind = kFdFile;

58
libc/calls/reservefd.c
View file

@ -29,33 +29,38 @@
#include "libc/str/str.h"
#include "libc/sysv/errfuns.h"
// XXX: until we can add read locks to all the code that uses g_fds.p
// (right now we only have write locks) we need to keep old copies
// of g_fds.p around after it's been extended, so that threads
// which are using an fd they de facto own can continue reading
static void FreeOldFdsArray(void *p) {
weaken(free)(p);
}
/**
* Grows file descriptor array memory if needed.
*/
int __ensurefds_unlocked(int fd) {
size_t n1, n2;
struct Fd *p1, *p2;
if (fd < g_fds.n) return fd;
STRACE("__ensurefds(%d) extending", fd);
if (!weaken(malloc)) return emfile();
p1 = g_fds.p;
n1 = g_fds.n;
if (fd >= n1) {
STRACE("__ensurefds(%d) extending", fd);
if (weaken(malloc)) {
// TODO(jart): we need a semaphore for this
p1 = g_fds.p;
n2 = fd + (fd >> 1);
if ((p2 = weaken(malloc)(n2 * sizeof(*p1)))) {
memcpy(p2, p1, n1 * sizeof(*p1));
g_fds.p = p2;
g_fds.n = n2;
if (p1 != g_fds.__init_p) {
__cxa_atexit(free, p1, 0);
}
} else {
fd = enomem();
}
} else {
fd = emfile();
}
if (p1 == g_fds.__init_p) {
if (!(p2 = weaken(malloc)(sizeof(g_fds.__init_p)))) return -1;
memcpy(p2, p1, sizeof(g_fds.__init_p));
g_fds.p = p1 = p2;
}
n2 = n1;
while (n2 <= fd) n2 *= 2;
if (!(p2 = weaken(malloc)(n2 * sizeof(*p1)))) return -1;
__cxa_atexit(FreeOldFdsArray, p1, 0);
memcpy(p2, p1, n1 * sizeof(*p1));
bzero(p2 + n1, (p2 + n2) - (p2 + n1));
g_fds.p = p2;
g_fds.n = n2;
return fd;
}
@ -74,7 +79,7 @@ int __ensurefds(int fd) {
*/
int __reservefd_unlocked(int start) {
int fd;
for (fd = g_fds.f; fd < g_fds.n; ++fd) {
for (fd = MAX(start, g_fds.f); fd < g_fds.n; ++fd) {
if (!g_fds.p[fd].kind) {
break;
}
@ -100,10 +105,10 @@ int __reservefd(int start) {
* Closes non-stdio file descriptors to free dynamic memory.
*/
static void FreeFds(void) {
int i;
NTTRACE("FreeFds()");
int i, keep = 3;
STRACE("FreeFds()");
_spinlock(&__fds_lock);
for (i = 3; i < g_fds.n; ++i) {
for (i = keep; i < g_fds.n; ++i) {
if (g_fds.p[i].kind) {
_spunlock(&__fds_lock);
close(i);
@ -111,8 +116,11 @@ static void FreeFds(void) {
}
}
if (g_fds.p != g_fds.__init_p) {
memcpy(g_fds.__init_p, g_fds.p, sizeof(*g_fds.p) * 3);
weaken(free)(g_fds.p);
bzero(g_fds.__init_p, sizeof(g_fds.__init_p));
memcpy(g_fds.__init_p, g_fds.p, sizeof(*g_fds.p) * keep);
if (weaken(free)) {
weaken(free)(g_fds.p);
}
g_fds.p = g_fds.__init_p;
g_fds.n = ARRAYLEN(g_fds.__init_p);
}

10
libc/calls/sigchld-nt.c
View file

@ -21,6 +21,7 @@
#include "libc/calls/sig.internal.h"
#include "libc/calls/strace.internal.h"
#include "libc/dce.h"
#include "libc/intrin/spinlock.h"
#include "libc/nt/enum/wait.h"
#include "libc/nt/runtime.h"
#include "libc/nt/synchronization.h"
@ -38,7 +39,10 @@ void _check_sigchld(void) {
int pids[64];
uint32_t i, n;
int64_t handles[64];
if (!(n = __sample_pids(pids, handles, true))) return;
_spinlock(&__fds_lock);
n = __sample_pids(pids, handles, true);
_spunlock(&__fds_lock);
if (!n) return;
i = WaitForMultipleObjects(n, handles, false, 0);
if (i == kNtWaitTimeout) return;
if (i == kNtWaitFailed) {
@ -53,8 +57,10 @@ void _check_sigchld(void) {
if (__sighandflags[SIGCHLD] & SA_NOCLDWAIT) {
STRACE("SIGCHILD SA_NOCLDWAIT fd=%d handle=%ld", pids[i], handles[i]);
CloseHandle(handles[i]);
__releasefd_unlocked(pids[i]);
__releasefd(pids[i]);
}
_spinlock(&__fds_lock);
g_fds.p[pids[i]].zombie = true;
_spunlock(&__fds_lock);
__sig_add(SIGCHLD, CLD_EXITED);
}

14
libc/intrin/_spinlock_debug_4.c
View file

@ -16,24 +16,30 @@
│ TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR │
│ PERFORMANCE OF THIS SOFTWARE. │
╚─────────────────────────────────────────────────────────────────────────────*/
#include "libc/bits/weaken.h"
#include "libc/calls/calls.h"
#include "libc/intrin/kprintf.h"
#include "libc/intrin/lockcmpxchgp.h"
#include "libc/intrin/spinlock.h"
#include "libc/log/log.h"
#include "libc/nexgen32e/rdtsc.h"
#include "libc/runtime/internal.h"
#include "libc/time/clockstonanos.internal.h"
void _spinlock_debug_4(void *lockptr, const char *lockname, const char *file,
int line, const char *func) {
privileged void _spinlock_debug_4(int *lock, const char *lockname,
const char *file, int line,
const char *func) {
unsigned i;
int me, owner;
uint64_t ts1, ts2;
int me, owner, *lock = lockptr;
me = gettid();
owner = 0;
if (!_lockcmpxchgp(lock, &owner, me)) {
if (owner == me) {
kprintf("%s:%d: warning: lock re-entry on %s in %s()\n", file, line,
kprintf("%s:%d: error: lock re-entry on %s in %s()\n", file, line,
lockname, func);
if (weaken(__die)) weaken(__die)();
__restorewintty();
_Exit(1);
}
i = 0;

50
libc/intrin/_spinlock_debug_1.c → libc/intrin/_trylock_debug_4.c
View file

@ -16,41 +16,27 @@
│ TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR │
│ PERFORMANCE OF THIS SOFTWARE. │
╚─────────────────────────────────────────────────────────────────────────────*/
#include "libc/calls/calls.h"
#include "libc/bits/weaken.h"
#include "libc/intrin/kprintf.h"
#include "libc/intrin/lockcmpxchgp.h"
#include "libc/intrin/spinlock.h"
#include "libc/nexgen32e/rdtsc.h"
#include "libc/time/clockstonanos.internal.h"
#include "libc/log/log.h"
#include "libc/runtime/internal.h"
#include "libc/runtime/runtime.h"
void _spinlock_debug_1(void *lockptr, const char *lockname, const char *file,
int line, const char *func) {
unsigned i;
uint64_t ts1, ts2;
int me, owner, *lock = lockptr;
me = gettid();
owner = 0;
if (!_lockcmpxchgp(lock, &owner, me)) {
if (owner == me) {
kprintf("%s:%d: warning: possible re-entry on lock %s in %s()\n", file,
line, lockname, func);
}
i = 0;
ts1 = rdtsc();
for (;;) {
owner = 0;
if (_lockcmpxchgp(lock, &owner, me)) break;
ts2 = rdtsc();
if (ClocksToNanos(ts1, ts2) > 1000000000ul) {
ts1 = ts2;
kprintf("%s:%d: warning: slow lock on %s in %s()\n", file, line,
lockname, func);
}
if (++i & 7) {
__builtin_ia32_pause();
} else {
sched_yield();
}
}
privileged int _trylock_debug_4(int *lock, const char *lockname,
const char *file, int line, const char *func) {
int owner = 0;
int me = gettid();
if (_lockcmpxchgp(lock, &owner, me)) {
return 0;
} else if (owner != me) {
return owner;
} else {
kprintf("%s:%d: error: lock re-entry on %s in %s()\n", file, line, lockname,
func);
if (weaken(__die)) weaken(__die)();
__restorewintty();
_Exit(1);
}
}

8
libc/intrin/gettid.greg.c
View file

@ -20,6 +20,10 @@
#include "libc/dce.h"
#include "libc/nexgen32e/threaded.h"
#include "libc/nt/thread.h"
#include "libc/nt/thunk/msabi.h"
#include "libc/runtime/internal.h"
__msabi extern typeof(GetCurrentThreadId) *const __imp_GetCurrentThreadId;
/**
* Returns current thread id.
@ -38,7 +42,7 @@ privileged int gettid(void) {
}
if (IsWindows()) {
return GetCurrentThreadId();
return __imp_GetCurrentThreadId();
}
if (IsLinux()) {
@ -83,5 +87,5 @@ privileged int gettid(void) {
return wut; // narrowing intentional
}
return getpid();
return __pid;
}

15
libc/intrin/intrin.mk
View file

@ -74,10 +74,23 @@ o/$(MODE)/libc/intrin/kprintf.greg.o: \
-fno-sanitize=all \
-fno-stack-protector
# we can't use compiler magic because:
# spinlocks are called very early in initialization
# e.g. __cxa_atexit()
o/$(MODE)/libc/intrin/gettid.greg.o \
o/$(MODE)/libc/intrin/_trylock_debug_4.o \
o/$(MODE)/libc/intrin/_spinlock_debug_4.o: \
OVERRIDE_CFLAGS += \
-fwrapv \
-x-no-pg \
-mno-fentry \
-ffreestanding \
-fno-sanitize=all \
-fno-stack-protector
o/$(MODE)/libc/intrin/tls.greg.o \
o/$(MODE)/libc/intrin/exit.greg.o \
o/$(MODE)/libc/intrin/exit1.greg.o \
o/$(MODE)/libc/intrin/gettid.greg.o \
o/$(MODE)/libc/intrin/getenv.greg.o \
o/$(MODE)/libc/intrin/assertfail.greg.o \
o/$(MODE)/libc/intrin/describeiov.greg.o \

28
libc/intrin/once.h
View file

@ -2,20 +2,20 @@
#define COSMOPOLITAN_LIBC_INTRIN_ONCE_H_
#include "libc/intrin/spinlock.h"
#define _once(x) \
({ \
typeof(x) oncerc; \
static bool once; \
static typeof(oncerc) onceresult; \
_Alignas(64) static char oncelock; \
_spinlock(&oncelock); \
if (once) { \
oncerc = onceresult; \
} else { \
oncerc = onceresult = x; \
} \
_spunlock(&oncelock); \
oncerc; \
#define _once(x) \
({ \
typeof(x) oncerc; \
static bool once; \
static typeof(oncerc) onceresult; \
_Alignas(64) static int oncelock; \
_spinlock(&oncelock); \
if (once) { \
oncerc = onceresult; \
} else { \
oncerc = onceresult = x; \
} \
_spunlock(&oncelock); \
oncerc; \
})
#endif /* COSMOPOLITAN_LIBC_INTRIN_ONCE_H_ */

7
libc/intrin/releasefd.c
View file

@ -20,13 +20,6 @@
#include "libc/intrin/spinlock.h"
#include "libc/macros.internal.h"
void __releasefd_unlocked(int fd) {
if (0 <= fd && fd < g_fds.n) {
g_fds.p[fd].kind = 0;
g_fds.f = MIN(fd, g_fds.f);
}
}
void __releasefd(int fd) {
_spinlock(&__fds_lock);
__releasefd_unlocked(fd);

27
libc/intrin/releasefd_unlocked.c Normal file
View file

@ -0,0 +1,27 @@
/*-*- mode:c;indent-tabs-mode:nil;c-basic-offset:2;tab-width:8;coding:utf-8 -*-│
│vi: set net ft=c ts=2 sts=2 sw=2 fenc=utf-8 :vi│
╞══════════════════════════════════════════════════════════════════════════════╡
│ Copyright 2022 Justine Alexandra Roberts Tunney │
│ │
│ Permission to use, copy, modify, and/or distribute this software for │
│ any purpose with or without fee is hereby granted, provided that the │
│ above copyright notice and this permission notice appear in all copies. │
│ │
│ THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL │
│ WARRANTIES WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED │
│ WARRANTIES OF MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE │
│ AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL │
│ DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR │
│ PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER │
│ TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR │
│ PERFORMANCE OF THIS SOFTWARE. │
╚─────────────────────────────────────────────────────────────────────────────*/
#include "libc/calls/internal.h"
#include "libc/macros.internal.h"
void __releasefd_unlocked(int fd) {
if (0 <= fd && fd < g_fds.n) {
g_fds.p[fd].kind = 0;
g_fds.f = MIN(fd, g_fds.f);
}
}

1
libc/intrin/sched_yield.S
View file

@ -19,6 +19,7 @@
#include "libc/dce.h"
#include "libc/sysv/consts/nr.h"
#include "libc/macros.internal.h"
.privileged
// Asks kernel to let other threads be scheduled.
//

54
libc/intrin/spinlock.h
View file

@ -4,6 +4,10 @@
#include "libc/calls/calls.h"
#include "libc/intrin/lockcmpxchg.h"
#include "libc/intrin/lockcmpxchgp.h"
/*───────────────────────────────────────────────────────────────────────────│─╗
│ cosmopolitan § spinlocks ─╬─│┼
╚────────────────────────────────────────────────────────────────────────────│─╝
privileged unsophisticated locking subroutines */
#if IsModeDbg() && !defined(_SPINLOCK_DEBUG)
#define _SPINLOCK_DEBUG
@ -12,15 +16,27 @@
#if defined(_SPINLOCK_DEBUG)
#define _spinlock(lock) _spinlock_ndebug(lock)
#define _spinlock_ndebug(lock) _spinlock_cooperative(lock)
#define _trylock(lock) _trylock_debug(lock)
#define _seizelock(lock) _seizelock_impl(lock, gettid())
#elif defined(TINY)
#define _spinlock(lock) _spinlock_tiny(lock)
#define _spinlock_ndebug(lock) _spinlock_tiny(lock)
#define _trylock(lock) _trylock_inline(lock)
#define _seizelock(lock) _seizelock_impl(lock, 1)
#else
#define _spinlock(lock) _spinlock_cooperative(lock)
#define _spinlock_ndebug(lock) _spinlock_cooperative(lock)
#define _trylock(lock) _trylock_inline(lock)
#define _seizelock(lock) _seizelock_impl(lock, 1)
#endif
#define _trylock(lock) __atomic_test_and_set(lock, __ATOMIC_SEQ_CST)
#define _trylock_inline(lock) __atomic_test_and_set(lock, __ATOMIC_SEQ_CST)
#define _trylock_debug(lock) \
_trylock_debug_4(lock, #lock, __FILE__, __LINE__, __FUNCTION__)
#define _spinlock_debug(lock) \
_spinlock_debug_4(lock, #lock, __FILE__, __LINE__, __FUNCTION__)
#define _spunlock(lock) \
do { \
@ -29,19 +45,19 @@
__atomic_store(__lock, &__x, __ATOMIC_RELAXED); \
} while (0)
#define _seizelock(lock) \
#define _seizelock_impl(lock, value) \
do { \
autotype(lock) __lock = (lock); \
typeof(*__lock) __x = 1; \
typeof(*__lock) __x = (value); \
__atomic_store(__lock, &__x, __ATOMIC_RELEASE); \
} while (0)
#define _spinlock_tiny(lock) \
do { \
autotype(lock) __lock = (lock); \
while (_trylock(__lock)) { \
__builtin_ia32_pause(); \
} \
#define _spinlock_tiny(lock) \
do { \
autotype(lock) __lock = (lock); \
while (_trylock_inline(__lock)) { \
__builtin_ia32_pause(); \
} \
} while (0)
#define _spinlock_cooperative(lock) \
@ -51,7 +67,7 @@
int __tries = 0; \
for (;;) { \
__atomic_load(__lock, &__x, __ATOMIC_RELAXED); \
if (!__x && !_trylock(__lock)) { \
if (!__x && !_trylock_inline(__lock)) { \
break; \
} else if (++__tries & 7) { \
__builtin_ia32_pause(); \
@ -61,21 +77,7 @@
} \
} while (0)
void _spinlock_debug_1(void *, const char *, const char *, int, const char *);
void _spinlock_debug_4(void *, const char *, const char *, int, const char *);
#define _spinlock_debug(lock) \
do { \
switch (sizeof(*(lock))) { \
case 1: \
_spinlock_debug_1(lock, #lock, __FILE__, __LINE__, __FUNCTION__); \
break; \
case 4: \
_spinlock_debug_4(lock, #lock, __FILE__, __LINE__, __FUNCTION__); \
break; \
default: \
assert(!"unsupported size"); \
} \
} while (0)
int _trylock_debug_4(int *, const char *, const char *, int, const char *);
void _spinlock_debug_4(int *, const char *, const char *, int, const char *);
#endif /* COSMOPOLITAN_LIBC_INTRIN_SPINLOCK_H_ */

15
libc/intrin/virtualprotect.c
View file

@ -24,6 +24,11 @@
__msabi extern typeof(VirtualProtect) *const __imp_VirtualProtect;
static const char *DescribeVpFlags(uint32_t *x) {
if (!x) return "n/a";
return DescribeNtPageFlags(*x);
}
/**
* Protects memory on the New Technology.
* @note this wrapper takes care of ABI, STRACE(), and __winerr()
@ -34,13 +39,9 @@ textwindows bool32 VirtualProtect(void *lpAddress, uint64_t dwSize,
bool32 bOk;
char oldbuf[64];
bOk = __imp_VirtualProtect(lpAddress, dwSize, flNewProtect, lpflOldProtect);
if (bOk) {
__stpcpy(oldbuf, DescribeNtPageFlags(*lpflOldProtect));
} else {
__winerr();
__stpcpy(oldbuf, "n/a");
}
if (!bOk) __winerr();
NTTRACE("VirtualProtect(%p, %'zu, %s, [%s]) → %hhhd% m", lpAddress, dwSize,
DescribeNtPageFlags(flNewProtect), oldbuf, bOk);
DescribeNtPageFlags(flNewProtect),
DescribeVpFlags(bOk ? lpflOldProtect : 0), bOk);
return bOk;
}

4
libc/nexgen32e/threaded.h
View file

@ -23,9 +23,9 @@ void __install_tls(char[hasatleast 64]);
static noasan inline char *__get_tls(void) {
char *tib, *lin = (char *)0x30;
if (IsLinux() || IsFreebsd() || IsNetbsd() || IsOpenbsd()) {
asm("mov\t%%fs:(%1),%0" : "=a"(tib) : "r"(lin));
asm("mov\t%%fs:(%1),%0" : "=a"(tib) : "r"(lin) : "memory");
} else {
asm("mov\t%%gs:(%1),%0" : "=a"(tib) : "r"(lin));
asm("mov\t%%gs:(%1),%0" : "=a"(tib) : "r"(lin) : "memory");
if (IsWindows()) {
tib = *(char **)(tib + 0x1480 + __tls_index * 8);
}

2
libc/rand/rand64.c
View file

@ -31,7 +31,7 @@
static int thepid;
static uint128_t thepool;
_Alignas(64) static char rand64_lock;
_Alignas(64) static int rand64_lock;
/**
* Returns nondeterministic random data.

10
libc/runtime/cosmo.S
View file

@ -135,6 +135,16 @@ cosmo: push %rbp
.init.end 306,_init_ftrace
#endif
#if IsAsan()
.init.start 306,_init_symbols
push %rdi
push %rsi
call __init_symbols
pop %rsi
pop %rdi
.init.end 306,_init_symbols
#endif
#if IsModeDbg()
#ifdef SYSDEBUG
.init.start 307,_init_printargs

2
libc/runtime/getsymboltable.c
View file

@ -30,7 +30,7 @@
#include "libc/zip.h"
#include "libc/zipos/zipos.internal.h"
static char g_lock;
static int g_lock;
hidden struct SymbolTable *__symtab; // for kprintf
/**

1
libc/runtime/hook.greg.c
View file

@ -59,6 +59,7 @@ privileged noinstrument noasan int __hook(void *ifunc,
intptr_t kMcount = (intptr_t)&mcount;
intptr_t kProgramCodeStart = (intptr_t)_ereal;
intptr_t kPrivilegedStart = (intptr_t)__privileged_addr;
if (!symbols) return -1;
__morph_begin();
for (i = 0; i < symbols->count; ++i) {
if (symbols->addr_base + symbols->symbols[i].x < kProgramCodeStart) {

30
libc/runtime/symbols.c Normal file
View file

@ -0,0 +1,30 @@
/*-*- mode:c;indent-tabs-mode:nil;c-basic-offset:2;tab-width:8;coding:utf-8 -*-│
│vi: set net ft=c ts=2 sts=2 sw=2 fenc=utf-8 :vi│
╞══════════════════════════════════════════════════════════════════════════════╡
│ Copyright 2022 Justine Alexandra Roberts Tunney │
│ │
│ Permission to use, copy, modify, and/or distribute this software for │
│ any purpose with or without fee is hereby granted, provided that the │
│ above copyright notice and this permission notice appear in all copies. │
│ │
│ THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL │
│ WARRANTIES WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED │
│ WARRANTIES OF MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE │
│ AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL │
│ DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR │
│ PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER │
│ TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR │
│ PERFORMANCE OF THIS SOFTWARE. │
╚─────────────────────────────────────────────────────────────────────────────*/
#include "libc/bits/weaken.h"
#include "libc/dce.h"
#include "libc/log/backtrace.internal.h"
#include "libc/log/log.h"
#include "libc/runtime/runtime.h"
#include "libc/runtime/symbols.internal.h"
void __init_symbols(void) {
if (__strace || (IsAsan() && weaken(__die))) {
GetSymbolTable();
}
}

10
libc/sock/accept-nt.c
View file

@ -19,6 +19,7 @@
#include "libc/calls/internal.h"
#include "libc/calls/sig.internal.h"
#include "libc/intrin/kprintf.h"
#include "libc/intrin/spinlock.h"
#include "libc/mem/mem.h"
#include "libc/nt/files.h"
#include "libc/nt/struct/pollfd.h"
@ -44,7 +45,9 @@ textwindows int sys_accept_nt(struct Fd *fd, void *addr, uint32_t *addrsize,
for (;;) {
if (!WSAPoll(&(struct sys_pollfd_nt){fd->handle, POLLIN}, 1,
__SIG_POLLING_INTERVAL_MS)) {
if (_check_interrupts(true, g_fds.p)) return eintr();
if (_check_interrupts(true, g_fds.p)) {
return eintr();
}
continue;
}
if ((h = WSAAccept(fd->handle, addr, (int32_t *)addrsize, 0, 0)) != -1) {
@ -54,7 +57,8 @@ textwindows int sys_accept_nt(struct Fd *fd, void *addr, uint32_t *addrsize,
if ((!(flags & SOCK_NONBLOCK) ||
__sys_ioctlsocket_nt(h, FIONBIO, (uint32_t[]){1}) != -1) &&
(sockfd2 = calloc(1, sizeof(struct SockFd)))) {
if ((client = __reservefd(-1)) != -1) {
_spinlock(&__fds_lock);
if ((client = __reservefd_unlocked(-1)) != -1) {
sockfd2->family = sockfd->family;
sockfd2->type = sockfd->type;
sockfd2->protocol = sockfd->protocol;
@ -63,8 +67,10 @@ textwindows int sys_accept_nt(struct Fd *fd, void *addr, uint32_t *addrsize,
g_fds.p[client].mode = 0140666;
g_fds.p[client].handle = h;
g_fds.p[client].extra = (uintptr_t)sockfd2;
_spunlock(&__fds_lock);
return client;
}
_spunlock(&__fds_lock);
free(sockfd2);
}
__sys_closesocket_nt(h);

6
libc/sock/epoll.c
View file

@ -36,6 +36,7 @@
#include "libc/calls/internal.h"
#include "libc/dce.h"
#include "libc/errno.h"
#include "libc/intrin/spinlock.h"
#include "libc/limits.h"
#include "libc/macros.internal.h"
#include "libc/mem/mem.h"
@ -1324,7 +1325,8 @@ static textwindows dontinline int sys_epoll_create1_nt(uint32_t flags) {
struct PortState *port_state;
struct TsTreeNode *tree_node;
if (wepoll_init() < 0) return -1;
if ((fd = __reservefd(-1)) == -1) return -1;
fd = __reservefd(-1);
if (fd == -1) return -1;
port_state = port_new(&ephnd);
if (!port_state) {
__releasefd(fd);
@ -1338,10 +1340,12 @@ static textwindows dontinline int sys_epoll_create1_nt(uint32_t flags) {
__releasefd(fd);
return -1;
}
_spinlock(&__fds_lock);
g_fds.p[fd].kind = kFdEpoll;
g_fds.p[fd].handle = ephnd;
g_fds.p[fd].flags = flags;
g_fds.p[fd].mode = 0140666;
_spunlock(&__fds_lock);
return fd;
}

3
libc/sock/socket-nt.c
View file

@ -44,7 +44,8 @@ textwindows int sys_socket_nt(int family, int type, int protocol) {
int64_t h;
struct SockFd *sockfd;
int fd, oflags, truetype;
if ((fd = __reservefd(-1)) == -1) return -1;
fd = __reservefd(-1);
if (fd == -1) return -1;
truetype = type & ~(SOCK_CLOEXEC | SOCK_NONBLOCK);
if ((h = WSASocket(family, truetype, protocol, NULL, 0,
kNtWsaFlagOverlapped)) != -1) {

55
libc/sock/socketpair-nt.c
View file

@ -30,10 +30,10 @@
#include "libc/sysv/errfuns.h"
textwindows int sys_socketpair_nt(int family, int type, int proto, int sv[2]) {
int64_t hpipe, h1, h2;
int reader, writer, oflags;
char16_t pipename[64];
uint32_t mode;
char16_t pipename[64];
int64_t hpipe, h1, h2;
int rc, reader, writer, oflags;
// Supports only AF_UNIX
if (family != AF_UNIX) {
@ -53,9 +53,13 @@ textwindows int sys_socketpair_nt(int family, int type, int proto, int sv[2]) {
}
CreatePipeName(pipename);
if ((reader = __reservefd(-1)) == -1) return -1;
if ((writer = __reservefd(-1)) == -1) {
__releasefd(reader);
_spinlock(&__fds_lock);
reader = __reservefd_unlocked(-1);
writer = __reservefd_unlocked(-1);
_spunlock(&__fds_lock);
if (reader == -1 || writer == -1) {
if (reader != -1) __releasefd(reader);
if (writer != -1) __releasefd(writer);
return -1;
}
if ((hpipe = CreateNamedPipe(
@ -68,28 +72,33 @@ textwindows int sys_socketpair_nt(int family, int type, int proto, int sv[2]) {
h1 = CreateFile(pipename, kNtGenericWrite | kNtGenericRead, 0,
&kNtIsInheritable, kNtOpenExisting, kNtFileFlagOverlapped, 0);
if (h1 == -1) {
CloseHandle(hpipe);
__releasefd(writer);
__releasefd(reader);
return -1;
}
_spinlock(&__fds_lock);
g_fds.p[reader].kind = kFdFile;
g_fds.p[reader].flags = oflags;
g_fds.p[reader].mode = 0140444;
g_fds.p[reader].handle = hpipe;
if (h1 != -1) {
g_fds.p[writer].kind = kFdFile;
g_fds.p[writer].flags = oflags;
g_fds.p[writer].mode = 0140222;
g_fds.p[writer].handle = h1;
g_fds.p[reader].kind = kFdFile;
g_fds.p[reader].flags = oflags;
g_fds.p[reader].mode = 0140444;
g_fds.p[reader].handle = hpipe;
g_fds.p[writer].kind = kFdFile;
g_fds.p[writer].flags = oflags;
g_fds.p[writer].mode = 0140222;
g_fds.p[writer].handle = h1;
sv[0] = reader;
sv[1] = writer;
rc = 0;
} else {
CloseHandle(hpipe);
__releasefd_unlocked(writer);
__releasefd_unlocked(reader);
rc = -1;
}
_spunlock(&__fds_lock);
sv[0] = reader;
sv[1] = writer;
return 0;
return rc;
}

2
libc/zipos/get.c
View file

@ -75,7 +75,7 @@ struct Zipos *__zipos_get(void) {
const char *progpath;
static struct Zipos zipos;
uint8_t *map, *base, *cdir;
_Alignas(64) static char lock;
_Alignas(64) static int lock;
_spinlock(&lock);
if (!once) {
sigfillset(&neu);

4
libc/zipos/open.c
View file

@ -25,6 +25,7 @@
#include "libc/calls/struct/stat.h"
#include "libc/dce.h"
#include "libc/errno.h"
#include "libc/intrin/spinlock.h"
#include "libc/macros.internal.h"
#include "libc/mem/mem.h"
#include "libc/nexgen32e/crc32.h"
@ -126,11 +127,14 @@ static int __zipos_load(struct Zipos *zipos, size_t cf, unsigned flags,
if (h->mem) {
if ((fd = IsWindows() ? __reservefd(-1) : dup(2)) != -1) {
if (__ensurefds(fd) != -1) {
_spinlock(&__fds_lock);
h->handle = g_fds.p[fd].handle;
g_fds.p[fd].kind = kFdZip;
g_fds.p[fd].handle = (intptr_t)h;
g_fds.p[fd].flags = flags | O_CLOEXEC;
g_fds.p[fd].mode = mode;
g_fds.p[fd].extra = 0;
_spunlock(&__fds_lock);
return fd;
}
close(fd);

25
test/libc/calls/open_test.c
View file

@ -16,8 +16,10 @@
│ TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR │
│ PERFORMANCE OF THIS SOFTWARE. │
╚─────────────────────────────────────────────────────────────────────────────*/
#include "libc/calls/internal.h"
#include "libc/dce.h"
#include "libc/errno.h"
#include "libc/macros.internal.h"
#include "libc/sysv/consts/o.h"
#include "libc/testlib/testlib.h"
#include "libc/x/x.h"
@ -88,3 +90,26 @@ TEST(open, testOpenExistingForAppendWriteOnly_seeksToEnd) {
EXPECT_STREQ("hello", buf);
EXPECT_SYS(0, 0, close(3));
}
int CountFds(void) {
int i, count;
for (count = i = 0; i < g_fds.n; ++i) {
if (g_fds.p[i].kind) {
++count;
}
}
return count;
}
TEST(open, lotsOfFds) {
if (!IsWindows()) return;
int i, n = 200;
ASSERT_SYS(0, 0, xbarf("hello.txt", "hello", -1));
for (i = 3; i < n; ++i) {
EXPECT_EQ(i, CountFds());
EXPECT_SYS(0, i, open("hello.txt", O_RDONLY));
}
for (i = 3; i < n; ++i) {
EXPECT_SYS(0, 0, close(i));
}
}

21
test/libc/runtime/clone_test.c
View file

@ -36,7 +36,8 @@
#include "libc/time/time.h"
char *stack, *tls;
int x, me, tid, thechilde, childetid;
int x, me, tid, *childetid;
_Atomic(int) thechilde;
void SetUp(void) {
x = 0;
@ -44,6 +45,7 @@ void SetUp(void) {
tls = calloc(1, 64);
__initialize_tls(tls);
*(int *)(tls + 0x3c) = 31337;
childetid = (int *)(tls + 0x0038);
ASSERT_NE(MAP_FAILED, (stack = mmap(0, GetStackSize(), PROT_READ | PROT_WRITE,
MAP_STACK | MAP_ANONYMOUS, -1, 0)));
}
@ -77,26 +79,25 @@ int CloneTest1(void *arg) {
ASSERT_EQ(31337, errno);
}
ASSERT_EQ(23, (intptr_t)arg);
thechilde = gettid();
ASSERT_NE(gettid(), getpid());
ASSERT_EQ(gettid(), childetid); // CLONE_CHILD_SETTID
ASSERT_EQ(gettid(), *childetid); // CLONE_CHILD_SETTID
return 0;
}
TEST(clone, test1) {
int ptid = 0;
_seizelock(&childetid);
*childetid = -1;
_seizelock(childetid);
ASSERT_NE(-1, (tid = clone(CloneTest1, stack, GetStackSize(),
CLONE_THREAD | CLONE_VM | CLONE_FS | CLONE_FILES |
CLONE_SIGHAND | CLONE_PARENT_SETTID |
CLONE_CHILD_SETTID | CLONE_CHILD_CLEARTID |
CLONE_SETTLS,
(void *)23, &ptid, tls, 64, &childetid)));
_spinlock(&childetid); // CLONE_CHILD_CLEARTID
(void *)23, &ptid, tls, 64, childetid)));
_spinlock(childetid); // CLONE_CHILD_CLEARTID
ASSERT_EQ(tid, ptid);
ASSERT_EQ(42, x);
ASSERT_NE(me, tid);
ASSERT_EQ(tid, thechilde);
ASSERT_EQ(0, errno);
errno = 31337;
ASSERT_EQ(31337, errno);
@ -115,13 +116,13 @@ int CloneTestSys(void *arg) {
TEST(clone, tlsSystemCallsErrno_wontClobberMainThreadBecauseTls) {
ASSERT_EQ(0, errno);
ASSERT_EQ(31337, *(int *)(tls + 0x3c));
_seizelock(&childetid);
_seizelock(childetid);
ASSERT_NE(-1, (tid = clone(CloneTestSys, stack, GetStackSize(),
CLONE_THREAD | CLONE_VM | CLONE_FS | CLONE_FILES |
CLONE_SIGHAND | CLONE_CHILD_SETTID |
CLONE_CHILD_CLEARTID | CLONE_SETTLS,
(void *)23, 0, tls, 64, &childetid)));
_spinlock(&childetid); // CLONE_CHILD_CLEARTID
(void *)23, 0, tls, 64, childetid)));
_spinlock(childetid); // CLONE_CHILD_CLEARTID
ASSERT_EQ(0, errno);
ASSERT_EQ(EFAULT, *(int *)(tls + 0x3c));
}