mirrors/cosmopolitan
1
0
Fork 0
mirror of https://github.com/jart/cosmopolitan.git synced 2025-01-31 03:27:39 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

Get rid of kmalloc()

Browse source 
This changes *NSYNC to allocate waiters on the stack so our locks don't
need to depend on dynamic memory. This make our runtiem simpler, and it
also fixes bugs with thread cancellation support.
This commit is contained in:
Justine Tunney 2023-09-11 21:34:53 -07:00
parent 77a7873057
commit a359de7893
No known key found for this signature in database
GPG key ID: BE714B4575D6E328
57 changed files with 405 additions and 472 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

7
ape/ape-m1.c
View file

@ -33,7 +33,7 @@
#define pagesz 16384
#define SYSLIB_MAGIC ('s' | 'l' << 8 | 'i' << 16 | 'b' << 24)
#define SYSLIB_VERSION 1
#define SYSLIB_VERSION 2
struct Syslib {
int magic;
@ -56,6 +56,9 @@ struct Syslib {
long (*dispatch_semaphore_signal)(dispatch_semaphore_t);
long (*dispatch_semaphore_wait)(dispatch_semaphore_t, dispatch_time_t);
dispatch_time_t (*dispatch_walltime)(const struct timespec *, int64_t);
/* v2 (2023-09-10) */
pthread_t (*pthread_self)(void);
void (*dispatch_release)(dispatch_semaphore_t);
};
#define ELFCLASS32 1
@ -829,6 +832,8 @@ int main(int argc, char **argv, char **envp) {
M->lib.dispatch_semaphore_signal = dispatch_semaphore_signal;
M->lib.dispatch_semaphore_wait = dispatch_semaphore_wait;
M->lib.dispatch_walltime = dispatch_walltime;
M->lib.pthread_self = pthread_self;
M->lib.dispatch_release = dispatch_release;
/* getenv("_") is close enough to at_execfn */
execfn = argc > 0 ? argv[0] : 0;

2
ape/ape.h
View file

@ -2,7 +2,7 @@
#define COSMOPOLITAN_APE_APE_H_
#define APE_VERSION_MAJOR 1
#define APE_VERSION_MINOR 7
#define APE_VERSION_MINOR 8
#define APE_VERSION_STR APE_VERSION_STR_(APE_VERSION_MAJOR, APE_VERSION_MINOR)
#define APE_VERSION_NOTE APE_VERSION_NOTE_(APE_VERSION_MAJOR, APE_VERSION_MINOR)

2
libc/calls/execve.c
View file

@ -71,7 +71,7 @@ int execve(const char *prog, char *const argv[], char *const envp[]) {
rc = _weaken(sys_pledge_linux)(__execpromises, __pledge_mode);
}
if (!rc) {
if (_weaken(__zipos_parseuri) &&
if (0 && _weaken(__zipos_parseuri) &&
(_weaken(__zipos_parseuri)(prog, &uri) != -1)) {
rc = _weaken(__zipos_open)(&uri, O_RDONLY | O_CLOEXEC);
if (rc != -1) {

11
libc/calls/fcntl-nt.c
View file

@ -25,12 +25,13 @@
#include "libc/calls/syscall_support-nt.internal.h"
#include "libc/calls/wincrash.internal.h"
#include "libc/errno.h"
#include "libc/intrin/kmalloc.h"
#include "libc/intrin/kprintf.h"
#include "libc/intrin/leaky.internal.h"
#include "libc/intrin/weaken.h"
#include "libc/limits.h"
#include "libc/log/backtrace.internal.h"
#include "libc/macros.internal.h"
#include "libc/mem/mem.h"
#include "libc/nt/createfile.h"
#include "libc/nt/enum/fileflagandattributes.h"
#include "libc/nt/enum/filelockflags.h"
@ -72,7 +73,7 @@ static textwindows struct FileLock *NewFileLock(void) {
fl = g_locks.free;
g_locks.free = fl->next;
} else {
fl = kmalloc(sizeof(*fl));
unassert((fl = _weaken(malloc)(sizeof(*fl))));
}
bzero(fl, sizeof(*fl));
fl->next = g_locks.list;
@ -80,6 +81,8 @@ static textwindows struct FileLock *NewFileLock(void) {
return fl;
}
IGNORE_LEAKS(NewFileLock)
static textwindows void FreeFileLock(struct FileLock *fl) {
fl->next = g_locks.free;
g_locks.free = fl;
@ -129,6 +132,10 @@ static textwindows int sys_fcntl_nt_lock(struct Fd *f, int fd, int cmd,
int64_t pos, off, len, end;
struct FileLock *fl, *ft, **flp;
if (!_weaken(malloc)) {
return enomem();
}
l = (struct flock *)arg;
len = l->l_len;
off = l->l_start;

2
libc/calls/ntspawn.c
View file

@ -87,8 +87,6 @@ textwindows int ntspawn(
block = NULL;
_init_sigchld();
if (__mkntpath(prog, prog16) == -1) return -1;
// we can't call malloc() because we're higher in the topological order
// we can't call kmalloc() because fork() calls this when kmalloc is locked
if ((handle = CreateFileMapping(-1, 0, pushpop(kNtPageReadwrite), 0,
sizeof(*block), 0)) &&
(block = MapViewOfFileEx(handle, kNtFileMapRead | kNtFileMapWrite, 0, 0,

1
libc/calls/syscall-sysv.internal.h
View file

@ -87,6 +87,7 @@ i32 sys_sem_close(i64);
i32 sys_sem_destroy(i64);
i32 sys_sem_getvalue(i64, u32 *);
i32 sys_sem_init(u32, i64 *);
i32 sys_sem_destroy(i64);
i32 sys_sem_open(const char *, int, u32, i64 *);
i32 sys_sem_post(i64);
i32 sys_sem_trywait(i64);

2
libc/calls/tcsetattr-nt.c
View file

@ -24,6 +24,7 @@
#include "libc/calls/termios.internal.h"
#include "libc/calls/ttydefaults.h"
#include "libc/dce.h"
#include "libc/errno.h"
#include "libc/intrin/describeflags.internal.h"
#include "libc/intrin/nomultics.internal.h"
#include "libc/intrin/strace.internal.h"
@ -261,6 +262,7 @@ textwindows int tcsetattr_nt(int fd, int opt, const struct termios *tio) {
__attribute__((__constructor__)) static void tcsetattr_nt_init(void) {
if (!getenv("TERM")) {
setenv("TERM", "xterm-256color", true);
errno = 0; // ignore malloc not linked
}
}

33
libc/calls/tkill.c
View file

@ -98,16 +98,43 @@ static dontinline textwindows int __tkill_nt(int tid, int sig,
}
}
static int __tkill_m1(int tid, int sig, struct CosmoTib *tib) {
struct PosixThread *pt = (struct PosixThread *)__get_tls()->tib_pthread;
static int __tkill_posix(int tid, int sig, struct CosmoTib *tib) {
// avoid lock when killing self
int me;
struct PosixThread *pt;
pt = (struct PosixThread *)__get_tls()->tib_pthread;
me = atomic_load_explicit(&__get_tls()->tib_tid, memory_order_relaxed);
if (tid == me && (!tib || tib == __get_tls())) {
return __syslib->pthread_kill(pt->next, sig);
}
// otherwise look for matching thread
struct Dll *e;
pthread_spin_lock(&_pthread_lock);
for (e = dll_first(_pthread_list); e; e = dll_next(_pthread_list, e)) {
enum PosixThreadStatus status;
struct PosixThread *pt = POSIXTHREAD_CONTAINER(e);
int rhs = atomic_load_explicit(&pt->tib->tib_tid, memory_order_acquire);
if (rhs <= 0 || tid != rhs) continue;
if (tib && tib != pt->tib) continue;
status = atomic_load_explicit(&pt->status, memory_order_acquire);
pthread_spin_unlock(&_pthread_lock);
if (status < kPosixThreadTerminated) {
return __syslib->pthread_kill(pt->next, sig);
} else {
return 0;
}
}
pthread_spin_unlock(&_pthread_lock);
return esrch();
}
// OpenBSD has an optional `tib` parameter for extra safety.
int __tkill(int tid, int sig, void *tib) {
int rc;
if (IsXnuSilicon()) {
return __tkill_m1(tid, sig, tib);
return __tkill_posix(tid, sig, tib);
} else if (IsLinux() || IsXnu() || IsFreebsd() || IsOpenbsd() || IsNetbsd()) {
rc = sys_tkill(tid, sig, tib);
} else if (IsWindows()) {

4
libc/intrin/asan.c
View file

@ -27,7 +27,6 @@
#include "libc/intrin/bits.h"
#include "libc/intrin/cmpxchg.h"
#include "libc/intrin/directmap.internal.h"
#include "libc/intrin/kmalloc.h"
#include "libc/intrin/kprintf.h"
#include "libc/intrin/leaky.internal.h"
#include "libc/intrin/likely.h"
@ -835,10 +834,9 @@ static __wur __asan_die_f *__asan_report(const void *addr, int size,
wint_t c;
signed char t;
uint64_t x, y, z;
char *p, *q, *buf, *base;
struct MemoryIntervals *m;
char buf[8192], *base, *q, *p = buf;
ftrace_enabled(-1);
p = buf = kmalloc(1024 * 1024);
kprintf("\n\e[J\e[1;31masan error\e[0m: %s %d-byte %s at %p shadow %p\n",
__asan_describe_access_poison(kind), size, message, addr,
SHADOW(addr));

2
libc/intrin/describeframe.c
View file

@ -43,8 +43,6 @@ static const char *GetFrameName(int x) {
return "g_fds";
} else if (IsZiposFrame(x)) {
return "zipos";
} else if (IsKmallocFrame(x)) {
return "kmalloc";
} else if (IsMemtrackFrame(x)) {
return "memtrack";
} else if (IsOldStackFrame(x)) {

95
libc/intrin/kmalloc.c
View file

@ -1,95 +0,0 @@
/*-*- mode:c;indent-tabs-mode:nil;c-basic-offset:2;tab-width:8;coding:utf-8 -*-│
│vi: set net ft=c ts=2 sts=2 sw=2 fenc=utf-8 :vi│
╞══════════════════════════════════════════════════════════════════════════════╡
│ Copyright 2022 Justine Alexandra Roberts Tunney │
│ │
│ Permission to use, copy, modify, and/or distribute this software for │
│ any purpose with or without fee is hereby granted, provided that the │
│ above copyright notice and this permission notice appear in all copies. │
│ │
│ THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL │
│ WARRANTIES WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED │
│ WARRANTIES OF MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE │
│ AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL │
│ DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR │
│ PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER │
│ TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR │
│ PERFORMANCE OF THIS SOFTWARE. │
╚─────────────────────────────────────────────────────────────────────────────*/
#include "libc/intrin/kmalloc.h"
#include "libc/assert.h"
#include "libc/atomic.h"
#include "libc/calls/syscall-sysv.internal.h"
#include "libc/dce.h"
#include "libc/intrin/asan.internal.h"
#include "libc/intrin/atomic.h"
#include "libc/intrin/describebacktrace.internal.h"
#include "libc/intrin/extend.internal.h"
#include "libc/log/libfatal.internal.h"
#include "libc/macros.internal.h"
#include "libc/runtime/memtrack.internal.h"
#include "libc/sysv/consts/map.h"
#include "libc/thread/thread.h"
#include "libc/thread/tls.h"
#define KMALLOC_ALIGN sizeof(intptr_t)
static struct {
char *endptr;
size_t total;
pthread_spinlock_t lock;
} g_kmalloc;
void __kmalloc_lock(void) {
pthread_spin_lock(&g_kmalloc.lock);
}
void __kmalloc_unlock(void) {
pthread_spin_unlock(&g_kmalloc.lock);
}
#ifdef _NOPL0
#define __kmalloc_lock() _NOPL0("__threadcalls", __kmalloc_lock)
#define __kmalloc_unlock() _NOPL0("__threadcalls", __kmalloc_unlock)
#endif
/**
* Allocates permanent memory.
*
* The code malloc() depends upon uses this function to allocate memory.
* The returned memory can't be freed, and leak detection is impossible.
* This function panics when memory isn't available.
*
* Memory returned by this function is aligned on the word size, and as
* such, kmalloc() shouldn't be used for vector operations.
*
* @return zero-initialized memory on success, or null w/ errno
* @raise ENOMEM if we require more vespene gas
*/
dontasan void *kmalloc(size_t size) {
char *p, *e;
size_t i, n, t;
n = ROUNDUP(size + (IsAsan() * 8), KMALLOC_ALIGN);
__kmalloc_lock();
t = g_kmalloc.total;
e = g_kmalloc.endptr;
i = t;
t += n;
p = (char *)kMemtrackKmallocStart;
if (!e) e = p;
if ((e = _extend(p, t, e, MAP_PRIVATE,
kMemtrackKmallocStart + kMemtrackKmallocSize))) {
g_kmalloc.endptr = e;
g_kmalloc.total = t;
} else {
p = 0;
}
__kmalloc_unlock();
if (p) {
unassert(!((intptr_t)(p + i) & (KMALLOC_ALIGN - 1)));
if (IsAsan()) __asan_poison(p + i + size, n - size, kAsanHeapOverrun);
return p + i;
} else {
return 0;
}
}

17
libc/intrin/kmalloc.h
View file

@ -1,17 +0,0 @@
#ifndef COSMOPOLITAN_LIBC_INTRIN_KMALLOC_H_
#define COSMOPOLITAN_LIBC_INTRIN_KMALLOC_H_
#ifdef _COSMO_SOURCE
#define kmalloc __kmalloc
#if !(__ASSEMBLER__ + __LINKER__ + 0)
COSMOPOLITAN_C_START_
void *kmalloc(size_t)
mallocesque attributeallocsize((1)) returnsaligned((8));
void __kmalloc_lock(void);
void __kmalloc_unlock(void);
COSMOPOLITAN_C_END_
#endif /* !(__ASSEMBLER__ + __LINKER__ + 0) */
#endif /* _COSMO_SOURCE */
#endif /* COSMOPOLITAN_LIBC_INTRIN_KMALLOC_H_ */

7
libc/intrin/leaky.internal.h
View file

@ -1,12 +1,17 @@
#ifndef COSMOPOLITAN_LIBC_INTRIN_LEAKY_INTERNAL_H_
#define COSMOPOLITAN_LIBC_INTRIN_LEAKY_INTERNAL_H_
#include "libc/dce.h"
#if !(__ASSEMBLER__ + __LINKER__ + 0)
COSMOPOLITAN_C_START_
#if IsAsan()
#define IGNORE_LEAKS(FUNC) \
__static_yoink("_leaky_start"); \
void *_leaky_##FUNC[] _Section(".piro.relo.sort.leaky.2." #FUNC \
",\"aw\",@init_array #") = {FUNC}
",\"aw\",@init_array #") = {FUNC};
#else
#define IGNORE_LEAKS(FUNC)
#endif
extern intptr_t _leaky_end[] __attribute__((__weak__));
extern intptr_t _leaky_start[] __attribute__((__weak__));

1
libc/intrin/pthread_mutex_trylock.c
View file

@ -19,6 +19,7 @@
#include "libc/calls/calls.h"
#include "libc/errno.h"
#include "libc/intrin/atomic.h"
#include "libc/intrin/kprintf.h"
#include "libc/intrin/weaken.h"
#include "libc/thread/thread.h"
#include "libc/thread/tls.h"

0
libc/thread/pthread_mutexattr_destroy.c → libc/intrin/pthread_mutexattr_destroy.c
View file

0
libc/thread/pthread_mutexattr_getpshared.c → libc/intrin/pthread_mutexattr_getpshared.c
View file

0
libc/thread/pthread_mutexattr_gettype.c → libc/intrin/pthread_mutexattr_gettype.c
View file

0
libc/thread/pthread_mutexattr_init.c → libc/intrin/pthread_mutexattr_init.c
View file

0
libc/thread/pthread_mutexattr_setpshared.c → libc/intrin/pthread_mutexattr_setpshared.c
View file

0
libc/thread/pthread_mutexattr_settype.c → libc/intrin/pthread_mutexattr_settype.c
View file

10
libc/intrin/putenv.c
View file

@ -17,11 +17,14 @@
│ PERFORMANCE OF THIS SOFTWARE. │
╚─────────────────────────────────────────────────────────────────────────────*/
#include "libc/intrin/getenv.internal.h"
#include "libc/intrin/kmalloc.h"
#include "libc/intrin/leaky.internal.h"
#include "libc/intrin/strace.internal.h"
#include "libc/intrin/weaken.h"
#include "libc/macros.internal.h"
#include "libc/mem/internal.h"
#include "libc/mem/mem.h"
#include "libc/runtime/runtime.h"
#include "libc/sysv/errfuns.h"
#define ToUpper(c) ((c) >= 'a' && (c) <= 'z' ? (c) - 'a' + 'A' : (c))
@ -40,7 +43,7 @@ static char **GrowEnviron(char **a) {
if (!a) a = environ;
n = a ? GetEnvironLen(a) : 0;
c = MAX(16ul, n) << 1;
if ((b = kmalloc(c * sizeof(char *)))) {
if (_weaken(malloc) && (b = _weaken(malloc)(c * sizeof(char *)))) {
if (a) {
for (p = b; *a;) {
*p++ = *a++;
@ -51,10 +54,13 @@ static char **GrowEnviron(char **a) {
capacity = c;
return b;
} else {
enomem();
return 0;
}
}
IGNORE_LEAKS(GrowEnviron)
int PutEnvImpl(char *s, bool overwrite) {
char **p;
struct Env e;

10
libc/intrin/setenv.c
View file

@ -16,9 +16,11 @@
│ TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR │
│ PERFORMANCE OF THIS SOFTWARE. │
╚─────────────────────────────────────────────────────────────────────────────*/
#include "libc/intrin/kmalloc.h"
#include "libc/intrin/leaky.internal.h"
#include "libc/intrin/strace.internal.h"
#include "libc/intrin/weaken.h"
#include "libc/mem/internal.h"
#include "libc/mem/mem.h"
#include "libc/runtime/runtime.h"
#include "libc/str/str.h"
#include "libc/sysv/errfuns.h"
@ -43,9 +45,13 @@ int setenv(const char *name, const char *value, int overwrite) {
}
n = strlen(name);
m = strlen(value);
if (!(s = kmalloc(n + 1 + m + 1))) return -1;
if (!_weaken(malloc) || !(s = _weaken(malloc)(n + 1 + m + 1))) {
return enomem();
}
memcpy(mempcpy(mempcpy(s, name, n), "=", 1), value, m + 1);
rc = PutEnvImpl(s, overwrite);
STRACE("setenv(%#s, %#s, %d) → %d% m", name, value, overwrite, rc);
return rc;
}
IGNORE_LEAKS(setenv)

9
libc/log/oncrash_amd64.c
View file

@ -32,7 +32,6 @@
#include "libc/intrin/atomic.h"
#include "libc/intrin/describebacktrace.internal.h"
#include "libc/intrin/describeflags.internal.h"
#include "libc/intrin/kmalloc.h"
#include "libc/intrin/kprintf.h"
#include "libc/intrin/strace.internal.h"
#include "libc/intrin/weaken.h"
@ -207,9 +206,9 @@ void ShowCrashReportHook(int, int, int, struct siginfo *, ucontext_t *);
relegated void ShowCrashReport(int err, int sig, struct siginfo *si,
ucontext_t *ctx) {
int i;
size_t n;
char *p;
char host[64];
char *p, *buf;
char buf[3000];
struct utsname names;
if (_weaken(ShowCrashReportHook)) {
ShowCrashReportHook(2, err, sig, si, ctx);
@ -223,9 +222,9 @@ relegated void ShowCrashReport(int err, int sig, struct siginfo *si,
uname(&names);
errno = err;
// TODO(jart): Buffer the WHOLE crash report with backtrace for atomic write.
npassert((p = buf = kmalloc((n = 1024 * 1024))));
p = buf;
p += ksnprintf(
p, n,
p, 10000,
"\n%serror%s: Uncaught %G (%s) on %s pid %d tid %d\n"
" %s\n"
" %s\n"

1
libc/runtime/fork-nt.c
View file

@ -68,7 +68,6 @@
extern int64_t __wincrashearly;
bool32 __onntconsoleevent(uint32_t);
void sys_setitimer_nt_reset(void);
void kmalloc_unlock(void);
static textwindows wontreturn void AbortFork(const char *func) {
#ifdef SYSDEBUG

7
libc/runtime/memtrack.internal.h
View file

@ -22,8 +22,6 @@ COSMOPOLITAN_C_START_
#define kMemtrackFdsSize (0x6feffffc0000 - kMemtrackFdsStart)
#define kMemtrackZiposStart 0x6fd000040000
#define kMemtrackZiposSize (0x6fdffffc0000 - kMemtrackZiposStart)
#define kMemtrackKmallocStart 0x6fc000040000
#define kMemtrackKmallocSize (0x6fcffffc0000 - kMemtrackKmallocStart)
#define kMemtrackGran (!IsAsan() ? FRAMESIZE : FRAMESIZE * 8)
struct MemoryInterval {
@ -132,11 +130,6 @@ forceinline pureconst bool IsZiposFrame(int x) {
x <= (int)((kMemtrackZiposStart + kMemtrackZiposSize - 1) >> 16);
}
forceinline pureconst bool IsKmallocFrame(int x) {
return (int)(kMemtrackKmallocStart >> 16) <= x &&
x <= (int)((kMemtrackKmallocStart + kMemtrackKmallocSize - 1) >> 16);
}
forceinline pureconst bool IsShadowFrame(int x) {
return 0x7fff <= x && x < 0x10008000;
}

2
libc/runtime/memtrack64.txt
View file

@ -1808,7 +1808,7 @@
6f900000-6f9fffff 64gb free
6fa00000-6fafffff 64gb free
6fb00000-6fbfffff 64gb free
6fc00004-6fcffffb 64gb kmalloc
6fc00004-6fcffffb 64gb free
6fd00004-6fdffffb 64gb zipos
6fe00004-6feffffb 64gb g_fds
6ff00004-70000003 64gb free

1
libc/runtime/syslib.internal.h
View file

@ -44,6 +44,7 @@ struct Syslib {
dispatch_time_t (*dispatch_walltime)(const struct timespec *, int64_t);
/* v2 (2023-09-10) */
long (*pthread_self)(void);
void (*dispatch_release)(dispatch_semaphore_t);
};
extern struct Syslib *__syslib;

3
libc/runtime/zipos-get.c
View file

@ -23,7 +23,6 @@
#include "libc/cosmo.h"
#include "libc/fmt/conv.h"
#include "libc/intrin/cmpxchg.h"
#include "libc/intrin/kmalloc.h"
#include "libc/intrin/promises.internal.h"
#include "libc/intrin/strace.internal.h"
#include "libc/macros.internal.h"
@ -91,7 +90,7 @@ static int __zipos_compare_names(const void *a, const void *b, void *c) {
static void __zipos_generate_index(struct Zipos *zipos) {
size_t c, i;
zipos->records = GetZipCdirRecords(zipos->cdir);
zipos->index = kmalloc(zipos->records * sizeof(size_t));
zipos->index = _mapanon(zipos->records * sizeof(size_t));
for (i = 0, c = GetZipCdirOffset(zipos->cdir); i < zipos->records;
++i, c += ZIP_CFILE_HDRSIZE(zipos->map + c)) {
zipos->index[i] = c;

45
libc/stdio/alloc.c
View file

@ -16,49 +16,26 @@
│ TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR │
│ PERFORMANCE OF THIS SOFTWARE. │
╚─────────────────────────────────────────────────────────────────────────────*/
#include "libc/intrin/atomic.h"
#include "libc/intrin/kmalloc.h"
#include "libc/mem/mem.h"
#include "libc/stdio/internal.h"
#include "libc/stdio/stdio.h"
#include "libc/str/str.h"
#include "libc/thread/thread.h"
static _Atomic(FILE *) __stdio_freelist;
FILE *__stdio_alloc(void) {
FILE *f;
f = atomic_load_explicit(&__stdio_freelist, memory_order_acquire);
while (f) {
if (atomic_compare_exchange_weak_explicit(
&__stdio_freelist, &f,
atomic_load_explicit((_Atomic(struct FILE *) *)&f->next,
memory_order_acquire),
memory_order_release, memory_order_relaxed)) {
atomic_store_explicit((_Atomic(struct FILE *) *)&f->next, 0,
memory_order_release);
break;
}
}
if (!f) {
f = kmalloc(sizeof(FILE));
}
if (f) {
f->lock._type = PTHREAD_MUTEX_RECURSIVE;
if ((f = calloc(1, sizeof(FILE)))) {
pthread_mutexattr_t attr;
pthread_mutexattr_init(&attr);
pthread_mutexattr_settype(&attr, PTHREAD_MUTEX_RECURSIVE);
pthread_mutex_init(&f->lock, &attr);
pthread_mutexattr_destroy(&attr);
f->dynamic = 1;
}
return f;
}
void __stdio_free(FILE *f) {
FILE *g;
bzero(f, sizeof(*f));
g = atomic_load_explicit(&__stdio_freelist, memory_order_acquire);
for (;;) {
atomic_store_explicit((_Atomic(struct FILE *) *)&f->next, g,
memory_order_release);
if (atomic_compare_exchange_weak_explicit(&__stdio_freelist, &g, f,
memory_order_release,
memory_order_relaxed)) {
break;
}
pthread_mutex_destroy(&f->lock);
if (f->dynamic) {
free(f);
}
}

31
libc/stdio/internal.h
View file

@ -9,21 +9,22 @@
COSMOPOLITAN_C_START_
struct FILE {
uint8_t bufmode; /* 0x00 _IOFBF, etc. (ignored if fd=-1) */
char noclose; /* 0x01 for fake dup() todo delete! */
uint32_t iomode; /* 0x04 O_RDONLY, etc. (ignored if fd=-1) */
int32_t state; /* 0x08 0=OK, -1=EOF, >0=errno */
int fd; /* 0x0c ≥0=fd, -1=closed|buffer */
uint32_t beg; /* 0x10 */
uint32_t end; /* 0x14 */
char *buf; /* 0x18 */
uint32_t size; /* 0x20 */
uint32_t nofree; /* 0x24 */
int pid; /* 0x28 */
char *getln; /* 0x30 */
pthread_mutex_t lock; /* 0x38 */
struct FILE *next; /* 0x48 */
char mem[BUFSIZ]; /* 0x50 */
uint8_t bufmode; /* _IOFBF, etc. (ignored if fd=-1) */
char noclose; /* for fake dup() todo delete! */
char dynamic; /* did malloc() create this object? */
uint32_t iomode; /* O_RDONLY, etc. (ignored if fd=-1) */
int32_t state; /* 0=OK, -1=EOF, >0=errno */
int fd; /* ≥0=fd, -1=closed|buffer */
uint32_t beg;
uint32_t end;
char *buf;
uint32_t size;
uint32_t nofree;
int pid;
char *getln;
pthread_mutex_t lock;
struct FILE *next;
char mem[BUFSIZ];
};
extern uint64_t g_rando;

13
libc/thread/pthread_atfork.c
View file

@ -17,13 +17,17 @@
│ PERFORMANCE OF THIS SOFTWARE. │
╚─────────────────────────────────────────────────────────────────────────────*/
#include "libc/assert.h"
#include "libc/atomic.h"
#include "libc/calls/state.internal.h"
#include "libc/cosmo.h"
#include "libc/dce.h"
#include "libc/errno.h"
#include "libc/intrin/atomic.h"
#include "libc/intrin/dll.h"
#include "libc/intrin/kmalloc.h"
#include "libc/intrin/leaky.internal.h"
#include "libc/mem/mem.h"
#include "libc/runtime/memtrack.internal.h"
#include "libc/runtime/runtime.h"
#include "libc/str/str.h"
#include "libc/thread/posixthread.internal.h"
#include "libc/thread/thread.h"
@ -61,11 +65,9 @@ void _pthread_onfork_prepare(void) {
pthread_spin_lock(&_pthread_lock);
__fds_lock();
__mmi_lock();
__kmalloc_lock();
}
void _pthread_onfork_parent(void) {
__kmalloc_unlock();
__mmi_unlock();
__fds_unlock();
pthread_spin_unlock(&_pthread_lock);
@ -85,7 +87,6 @@ void _pthread_onfork_child(void) {
atomic_store_explicit(&pt->cancelled, false, memory_order_relaxed);
// wipe core runtime locks
__kmalloc_unlock();
pthread_mutexattr_init(&attr);
pthread_mutexattr_settype(&attr, PTHREAD_MUTEX_RECURSIVE);
pthread_mutex_init(&__mmi_lock_obj, &attr);
@ -105,7 +106,7 @@ void _pthread_onfork_child(void) {
int _pthread_atfork(atfork_f prepare, atfork_f parent, atfork_f child) {
int rc;
struct AtFork *a;
if (!(a = kmalloc(sizeof(struct AtFork)))) return ENOMEM;
if (!(a = malloc(sizeof(struct AtFork)))) return ENOMEM;
a->f[0] = prepare;
a->f[1] = parent;
a->f[2] = child;
@ -118,3 +119,5 @@ int _pthread_atfork(atfork_f prepare, atfork_f parent, atfork_f child) {
rc = 0;
return rc;
}
IGNORE_LEAKS(_pthread_atfork)

2
libc/thread/tls.h
View file

@ -27,7 +27,7 @@ struct CosmoTib {
_Atomic(int32_t) tib_tid; /* 0x38 transitions -1 → tid → 0 */
int32_t tib_errno; /* 0x3c */
uint64_t tib_flags; /* 0x40 */
void *tib_nsync;
long __padding;
int tib_ftrace; /* inherited */
int tib_strace; /* inherited */
uint64_t tib_sigmask; /* inherited */

4
libc/x/xload.c
View file

@ -17,7 +17,7 @@
│ PERFORMANCE OF THIS SOFTWARE. │
╚─────────────────────────────────────────────────────────────────────────────*/
#include "libc/intrin/atomic.h"
#include "libc/intrin/kmalloc.h"
#include "libc/mem/mem.h"
#include "libc/runtime/internal.h"
#include "libc/x/x.h"
#include "third_party/zlib/zlib.h"
@ -44,7 +44,7 @@
void *xload(_Atomic(void *) *a, const void *p, size_t n, size_t m) {
void *r, *z;
if ((r = atomic_load_explicit(a, memory_order_acquire))) return r;
if (!(r = kmalloc(m))) return 0;
if (!(r = malloc(m))) return 0;
if (__inflate(r, m, p, n)) return 0;
z = 0;
if (!atomic_compare_exchange_strong_explicit(a, &z, r, memory_order_release,

2
test/libc/calls/sched_getaffinity_test.c
View file

@ -89,6 +89,7 @@ __attribute__((__constructor__)) static void init(void) {
}
}
#ifdef __x86_64__
TEST(sched_setaffinity, isInheritedAcrossExecve) {
cpu_set_t x;
CPU_ZERO(&x);
@ -103,6 +104,7 @@ TEST(sched_setaffinity, isInheritedAcrossExecve) {
EXPECT_TRUE(WIFEXITED(ws));
EXPECT_EQ(42, WEXITSTATUS(ws));
}
#endif /* __x86_64__ */
TEST(sched_getaffinity, getpid) {
cpu_set_t x, y;

1
test/libc/stdio/popen_test.c
View file

@ -28,7 +28,6 @@
#include "libc/mem/gc.h"
#include "libc/mem/mem.h"
#include "libc/runtime/runtime.h"
#include "libc/stdio/lock.internal.h"
#include "libc/stdio/rand.h"
#include "libc/stdio/stdio.h"
#include "libc/str/str.h"

2
test/libc/stdio/posix_spawn_test.c
View file

@ -251,7 +251,7 @@ void PosixSpawnWait(const char *prog) {
ASSERT_EQ(42, WEXITSTATUS(ws));
}
TEST(posix_spawn, bench) {
BENCH(posix_spawn, bench) {
long n = 128L * 1000 * 1000;
memset(gc(malloc(n)), -1, n);
creat("tiny64", 0755);

9
test/libc/thread/pthread_cancel_test.c
View file

@ -169,18 +169,17 @@ void *CondWaitDeferredWorker(void *arg) {
pthread_setcancelstate(PTHREAD_CANCEL_DEFERRED, 0);
ASSERT_EQ(0, pthread_mutex_lock(&mu));
ASSERT_EQ(ECANCELED, pthread_cond_timedwait(&cv, &mu, 0));
ASSERT_EQ(0, pthread_mutex_unlock(&mu));
return 0;
__builtin_trap();
}
TEST(pthread_cancel, condDeferredWait) {
TEST(pthread_cancel, condDeferredWait_reacquiresMutex) {
void *rc;
pthread_t th;
ASSERT_EQ(0, pthread_create(&th, 0, CondWaitDeferredWorker, 0));
pthread_cancel(th);
ASSERT_EQ(0, pthread_join(th, &rc));
ASSERT_EQ(PTHREAD_CANCELED, rc);
ASSERT_EQ(0, pthread_mutex_trylock(&mu));
ASSERT_EQ(EBUSY, pthread_mutex_trylock(&mu));
ASSERT_EQ(0, pthread_mutex_unlock(&mu));
}
@ -192,7 +191,7 @@ TEST(pthread_cancel, condDeferredWaitDelayed) {
pthread_cancel(th);
ASSERT_EQ(0, pthread_join(th, &rc));
ASSERT_EQ(PTHREAD_CANCELED, rc);
ASSERT_EQ(0, pthread_mutex_trylock(&mu));
ASSERT_EQ(EBUSY, pthread_mutex_trylock(&mu));
ASSERT_EQ(0, pthread_mutex_unlock(&mu));
}

4
third_party/finger/util.c vendored
View file

@ -93,7 +93,7 @@ static void userinfo(PERSON *pn, struct passwd *pw) {
pn->shell = strdup(pw->pw_shell);
/* make a private copy of gecos to munge */
strncpy(tbuf, pw->pw_gecos, TBUFLEN);
strlcpy(tbuf, pw->pw_gecos, TBUFLEN);
tbuf[TBUFLEN-1] = 0; /* ensure null termination */
bp = tbuf;
@ -176,7 +176,7 @@ match(struct passwd *pw, const char *user)
int i, j, ct, rv=0;
char *rname;
strncpy(tbuf, pw->pw_gecos, TBUFLEN);
strlcpy(tbuf, pw->pw_gecos, TBUFLEN);
tbuf[TBUFLEN-1] = 0; /* guarantee null termination */
p = tbuf;

25
third_party/nsync/README.cosmo vendored
View file

@ -15,9 +15,22 @@ ORIGIN
LOCAL CHANGES
- nsync_malloc_() is implemented as kmalloc()
- nsync_mu_semaphore uses Cosmopolitan Futexes
- block pthread cancellations in nsync_mu_lock_slow_
- support posix thread cancellations in nsync_cv_wait
- timespec api was so good that it's now part of libc
- linked list api was so good that it's now part of libc
- Time APIs were so good that they're now part of our libc
- Double linked list API was so good that it's now part of our libc
- Modified *NSYNC to allocate waiter objects on the stack. We need it
because we use *NSYNC mutexes to implement POSIX mutexes, which are
too low-level to safely depend on malloc, or even mmap in our case.
- Rewrote most of the semaphore and futex system call support code so
it works well with Cosmopolitan's fat runtime portability. *NSYNC's
unit test suite passes on all supported platforms. However the BSDs
currently appear to be overutilizing CPU time compared with others.
- Support POSIX thread cancellation. APIs that wait on condition vars
are now cancellation points. In PTHREAD_CANCEL_MASKED mode they may
return ECANCELED. In PTHREAD_CANCEL_DEFERRED mode the POSIX threads
library will unwind the stack to unlock any locks and free waiters.
On the other hand the *NSYNC APIs for mutexes will now safely block
thread cancellation, but you can still use *NSYNC notes to do that.

73
third_party/nsync/common.c vendored
View file

@ -16,10 +16,6 @@
│ limitations under the License. │
╚─────────────────────────────────────────────────────────────────────────────*/
#include "libc/intrin/dll.h"
#include "libc/intrin/kmalloc.h"
#include "libc/mem/mem.h"
#include "libc/runtime/runtime.h"
#include "libc/thread/thread.h"
#include "libc/thread/tls.h"
#include "third_party/nsync/atomic.h"
#include "third_party/nsync/atomic.internal.h"
@ -147,49 +143,8 @@ waiter *nsync_dll_waiter_samecond_ (struct Dll *e) {
/* -------------------------------- */
static struct Dll *free_waiters = NULL;
/* free_waiters points to a doubly-linked list of free waiter structs. */
static nsync_atomic_uint32_ free_waiters_mu; /* spinlock; protects free_waiters */
#define waiter_for_thread __get_tls()->tib_nsync
static void waiter_destroy (void *v) {
waiter *w = (waiter *) v;
/* Reset waiter_for_thread in case another thread-local variable reuses
the waiter in its destructor while the waiter is taken by the other
thread from free_waiters. This can happen as the destruction order
of thread-local variables can be arbitrary in some platform e.g.
POSIX. */
waiter_for_thread = NULL;
IGNORE_RACES_START ();
ASSERT ((w->flags & (WAITER_RESERVED|WAITER_IN_USE)) == WAITER_RESERVED);
w->flags &= ~WAITER_RESERVED;
nsync_spin_test_and_set_ (&free_waiters_mu, 1, 1, 0);
dll_make_first (&free_waiters, &w->nw.q);
ATM_STORE_REL (&free_waiters_mu, 0); /* release store */
IGNORE_RACES_END ();
}
/* Return a pointer to an unused waiter struct.
Ensures that the enclosed timer is stopped and its channel drained. */
waiter *nsync_waiter_new_ (void) {
struct Dll *q;
waiter *tw;
waiter *w;
tw = waiter_for_thread;
w = tw;
if (w == NULL || (w->flags & (WAITER_RESERVED|WAITER_IN_USE)) != WAITER_RESERVED) {
w = NULL;
nsync_spin_test_and_set_ (&free_waiters_mu, 1, 1, 0);
q = dll_first (free_waiters);
if (q != NULL) { /* If free list is non-empty, dequeue an item. */
dll_remove (&free_waiters, q);
w = DLL_WAITER (q);
}
ATM_STORE_REL (&free_waiters_mu, 0); /* release store */
if (w == NULL) { /* If free list was empty, allocate an item. */
w = (waiter *) kmalloc (sizeof (*w));
/* Initializes waiter struct. */
void nsync_waiter_init_ (waiter *w) {
w->tag = WAITER_TAG;
w->nw.tag = NSYNC_WAITER_TAG;
nsync_mu_semaphore_init (&w->sem);
@ -199,26 +154,14 @@ waiter *nsync_waiter_new_ (void) {
w->nw.flags = NSYNC_WAITER_FLAG_MUCV;
ATM_STORE (&w->remove_count, 0);
dll_init (&w->same_condition);
w->flags = 0;
}
if (tw == NULL) {
w->flags |= WAITER_RESERVED;
nsync_set_per_thread_waiter_ (w, &waiter_destroy);
waiter_for_thread = w;
}
}
w->flags |= WAITER_IN_USE;
return (w);
w->flags = WAITER_IN_USE;
}
/* Return an unused waiter struct *w to the free pool. */
void nsync_waiter_free_ (waiter *w) {
ASSERT ((w->flags & WAITER_IN_USE) != 0);
w->flags &= ~WAITER_IN_USE;
if ((w->flags & WAITER_RESERVED) == 0) {
nsync_spin_test_and_set_ (&free_waiters_mu, 1, 1, 0);
dll_make_first (&free_waiters, &w->nw.q);
ATM_STORE_REL (&free_waiters_mu, 0); /* release store */
/* Destroys waiter struct. */
void nsync_waiter_destroy_ (waiter *w) {
if (w->tag) {
nsync_mu_semaphore_destroy (&w->sem);
w->tag = 0;
}
}

8
third_party/nsync/common.internal.h vendored
View file

@ -244,12 +244,8 @@ waiter *nsync_dll_waiter_(struct Dll *e);
: DLL_CONTAINER(struct waiter_s, same_condition, e))
waiter *nsync_dll_waiter_samecond_(struct Dll *e);
/* Return a pointer to an unused waiter struct.
Ensures that the enclosed timer is stopped and its channel drained. */
waiter *nsync_waiter_new_(void);
/* Return an unused waiter struct *w to the free pool. */
void nsync_waiter_free_(waiter *w);
void nsync_waiter_init_(waiter *);
void nsync_waiter_destroy_(waiter *);
/* ---------- */

219
third_party/nsync/mem/nsync_cv.c vendored
View file

@ -16,6 +16,7 @@
│ limitations under the License. │
╚─────────────────────────────────────────────────────────────────────────────*/
#include "libc/calls/cp.internal.h"
#include "libc/errno.h"
#include "libc/intrin/dll.h"
#include "libc/str/str.h"
#include "libc/thread/thread.h"
@ -169,6 +170,95 @@ static void void_mu_unlock (void *mu) {
nsync_mu_unlock ((nsync_mu *) mu);
}
/* Memory needed for a single nsync_cv_wait_with_deadline_generic() call. */
struct nsync_cv_wait_with_deadline_s {
nsync_cv *pcv;
int sem_outcome;
int is_reader_mu;
uint32_t old_word;
uint32_t remove_count;
void *pmu;
void (*lock) (void *);
nsync_mu *cv_mu;
nsync_time abs_deadline;
nsync_note cancel_note;
waiter w;
};
/* Wait until awoken or timeout, or back out of wait if the thread is being cancelled. */
static int nsync_cv_wait_with_deadline_impl_ (struct nsync_cv_wait_with_deadline_s *c) {
int outcome = 0;
int attempts = 0;
IGNORE_RACES_START ();
while (ATM_LOAD_ACQ (&c->w.nw.waiting) != 0) { /* acquire load */
if (c->sem_outcome == 0) {
c->sem_outcome = nsync_sem_wait_with_cancel_ (&c->w, c->abs_deadline, c->cancel_note);
}
if (c->sem_outcome != 0 && ATM_LOAD (&c->w.nw.waiting) != 0) {
/* A timeout or cancellation occurred, and no wakeup.
Acquire *pcv's spinlock, and confirm. */
c->old_word = nsync_spin_test_and_set_ (&c->pcv->word, CV_SPINLOCK,
CV_SPINLOCK, 0);
/* Check that w wasn't removed from the queue after we
checked above, but before we acquired the spinlock.
The test of remove_count confirms that the waiter *w
is still governed by *pcv's spinlock; otherwise, some
other thread is about to set w.waiting==0. */
if (ATM_LOAD (&c->w.nw.waiting) != 0) {
if (c->remove_count == ATM_LOAD (&c->w.remove_count)) {
uint32_t old_value;
/* still in cv waiter queue */
/* Not woken, so remove *w from cv
queue, and declare a
timeout/cancellation. */
outcome = c->sem_outcome;
dll_remove (&c->pcv->waiters, &c->w.nw.q);
do {
old_value = ATM_LOAD (&c->w.remove_count);
} while (!ATM_CAS (&c->w.remove_count, old_value, old_value+1));
if (dll_is_empty (c->pcv->waiters)) {
c->old_word &= ~(CV_NON_EMPTY);
}
ATM_STORE_REL (&c->w.nw.waiting, 0); /* release store */
}
}
/* Release spinlock. */
ATM_STORE_REL (&c->pcv->word, c->old_word); /* release store */
}
if (ATM_LOAD (&c->w.nw.waiting) != 0) {
/* The delay here causes this thread ultimately to
yield to another that has dequeued this thread, but
has not yet set the waiting field to zero; a
cancellation or timeout may prevent this thread
from blocking above on the semaphore. */
attempts = nsync_spin_delay_ (attempts);
}
}
if (c->cv_mu != NULL && c->w.cv_mu == NULL) { /* waiter was moved to *pmu's queue, and woken. */
/* Requeue on *pmu using existing waiter struct; current thread
is the designated waker. */
nsync_mu_lock_slow_ (c->cv_mu, &c->w, MU_DESIG_WAKER, c->w.l_type);
} else {
/* Traditional case: We've woken from the cv, and need to reacquire *pmu. */
if (c->is_reader_mu) {
nsync_mu_rlock (c->cv_mu);
} else {
(*c->lock) (c->pmu);
}
}
nsync_waiter_destroy_ (&c->w);
IGNORE_RACES_END ();
return (outcome);
}
/* Handle POSIX thread DEFERRED mode cancellation. */
static void nsync_cv_wait_with_deadline_unwind_ (void *arg) {
struct nsync_cv_wait_with_deadline_s *c;
c = (struct nsync_cv_wait_with_deadline_s *)arg;
c->sem_outcome = ECANCELED;
nsync_cv_wait_with_deadline_impl_ (c);
}
/* Atomically release *pmu (which must be held on entry)
and block the calling thread on *pcv. Then wait until awakened by a
call to nsync_cv_signal() or nsync_cv_broadcast() (or a spurious wakeup), or by the time
@ -194,33 +284,33 @@ int nsync_cv_wait_with_deadline_generic (nsync_cv *pcv, void *pmu,
void (*lock) (void *), void (*unlock) (void *),
nsync_time abs_deadline,
nsync_note cancel_note) {
nsync_mu *cv_mu = NULL;
int is_reader_mu;
uint32_t old_word;
uint32_t remove_count;
int sem_outcome;
unsigned attempts;
int outcome = 0;
waiter *w;
int outcome;
struct nsync_cv_wait_with_deadline_s c;
IGNORE_RACES_START ();
BEGIN_CANCELLATION_POINT;
w = nsync_waiter_new_ ();
pthread_cleanup_push((void *)nsync_waiter_free_, w);
ATM_STORE (&w->nw.waiting, 1);
w->cond.f = NULL; /* Not using a conditional critical section. */
w->cond.v = NULL;
w->cond.eq = NULL;
nsync_waiter_init_ (&c.w);
c.abs_deadline = abs_deadline;
c.cancel_note = cancel_note;
c.cv_mu = NULL;
c.lock = lock;
c.pcv = pcv;
c.pmu = pmu;
ATM_STORE (&c.w.nw.waiting, 1);
c.w.cond.f = NULL; /* Not using a conditional critical section. */
c.w.cond.v = NULL;
c.w.cond.eq = NULL;
if (lock == &void_mu_lock ||
lock == (void (*) (void *)) &nsync_mu_lock ||
lock == (void (*) (void *)) &nsync_mu_rlock) {
cv_mu = (nsync_mu *) pmu;
c.cv_mu = (nsync_mu *) pmu;
}
w->cv_mu = cv_mu; /* If *pmu is an nsync_mu, record its address, else record NULL. */
is_reader_mu = 0; /* If true, an nsync_mu in reader mode. */
if (cv_mu == NULL) {
w->l_type = NULL;
c.w.cv_mu = c.cv_mu; /* If *pmu is an nsync_mu, record its address, else record NULL. */
c.is_reader_mu = 0; /* If true, an nsync_mu in reader mode. */
if (c.cv_mu == NULL) {
c.w.l_type = NULL;
} else {
uint32_t old_mu_word = ATM_LOAD (&cv_mu->word);
uint32_t old_mu_word = ATM_LOAD (&c.cv_mu->word);
int is_writer = (old_mu_word & MU_WHELD_IF_NON_ZERO) != 0;
int is_reader = (old_mu_word & MU_RHELD_IF_NON_ZERO) != 0;
if (is_writer) {
@ -228,95 +318,34 @@ int nsync_cv_wait_with_deadline_generic (nsync_cv *pcv, void *pmu,
nsync_panic_ ("mu held in reader and writer mode simultaneously "
"on entry to nsync_cv_wait_with_deadline()\n");
}
w->l_type = nsync_writer_type_;
c.w.l_type = nsync_writer_type_;
} else if (is_reader) {
w->l_type = nsync_reader_type_;
is_reader_mu = 1;
c.w.l_type = nsync_reader_type_;
c.is_reader_mu = 1;
} else {
nsync_panic_ ("mu not held on entry to nsync_cv_wait_with_deadline()\n");
}
}
/* acquire spinlock, set non-empty */
old_word = nsync_spin_test_and_set_ (&pcv->word, CV_SPINLOCK, CV_SPINLOCK|CV_NON_EMPTY, 0);
dll_make_last (&pcv->waiters, &w->nw.q);
remove_count = ATM_LOAD (&w->remove_count);
c.old_word = nsync_spin_test_and_set_ (&pcv->word, CV_SPINLOCK, CV_SPINLOCK|CV_NON_EMPTY, 0);
dll_make_last (&pcv->waiters, &c.w.nw.q);
c.remove_count = ATM_LOAD (&c.w.remove_count);
/* Release the spin lock. */
ATM_STORE_REL (&pcv->word, old_word|CV_NON_EMPTY); /* release store */
ATM_STORE_REL (&pcv->word, c.old_word|CV_NON_EMPTY); /* release store */
/* Release *pmu. */
if (is_reader_mu) {
nsync_mu_runlock (cv_mu);
if (c.is_reader_mu) {
nsync_mu_runlock (c.cv_mu);
} else {
(*unlock) (pmu);
}
/* wait until awoken or a timeout. */
sem_outcome = 0;
attempts = 0;
while (ATM_LOAD_ACQ (&w->nw.waiting) != 0) { /* acquire load */
if (sem_outcome == 0) {
sem_outcome = nsync_sem_wait_with_cancel_ (w, abs_deadline, cancel_note);
}
if (sem_outcome != 0 && ATM_LOAD (&w->nw.waiting) != 0) {
/* A timeout or cancellation occurred, and no wakeup.
Acquire *pcv's spinlock, and confirm. */
old_word = nsync_spin_test_and_set_ (&pcv->word, CV_SPINLOCK,
CV_SPINLOCK, 0);
/* Check that w wasn't removed from the queue after we
checked above, but before we acquired the spinlock.
The test of remove_count confirms that the waiter *w
is still governed by *pcv's spinlock; otherwise, some
other thread is about to set w.waiting==0. */
if (ATM_LOAD (&w->nw.waiting) != 0) {
if (remove_count == ATM_LOAD (&w->remove_count)) {
uint32_t old_value;
/* still in cv waiter queue */
/* Not woken, so remove *w from cv
queue, and declare a
timeout/cancellation. */
outcome = sem_outcome;
dll_remove (&pcv->waiters, &w->nw.q);
do {
old_value = ATM_LOAD (&w->remove_count);
} while (!ATM_CAS (&w->remove_count, old_value, old_value+1));
if (dll_is_empty (pcv->waiters)) {
old_word &= ~(CV_NON_EMPTY);
}
ATM_STORE_REL (&w->nw.waiting, 0); /* release store */
}
}
/* Release spinlock. */
ATM_STORE_REL (&pcv->word, old_word); /* release store */
}
if (ATM_LOAD (&w->nw.waiting) != 0) {
/* The delay here causes this thread ultimately to
yield to another that has dequeued this thread, but
has not yet set the waiting field to zero; a
cancellation or timeout may prevent this thread
from blocking above on the semaphore. */
attempts = nsync_spin_delay_ (attempts);
}
}
if (cv_mu != NULL && w->cv_mu == NULL) { /* waiter was moved to *pmu's queue, and woken. */
/* Requeue on *pmu using existing waiter struct; current thread
is the designated waker. */
nsync_mu_lock_slow_ (cv_mu, w, MU_DESIG_WAKER, w->l_type);
nsync_waiter_free_ (w);
} else {
/* Traditional case: We've woken from the cv, and need to reacquire *pmu. */
nsync_waiter_free_ (w);
if (is_reader_mu) {
nsync_mu_rlock (cv_mu);
} else {
(*lock) (pmu);
}
}
/* Wait until awoken or a timeout. */
c.sem_outcome = 0;
pthread_cleanup_push (nsync_cv_wait_with_deadline_unwind_, &c);
outcome = nsync_cv_wait_with_deadline_impl_ (&c);
pthread_cleanup_pop (0);
END_CANCELLATION_POINT;
IGNORE_RACES_END ();
return (outcome);
}

18
third_party/nsync/mem/nsync_mu_wait.c vendored
View file

@ -149,12 +149,12 @@ int nsync_mu_wait_with_deadline (nsync_mu *mu,
lock_type *l_type;
int first_wait;
int condition_is_true;
waiter *w;
waiter w[1];
int outcome;
/* Work out in which mode the lock is held. */
uint32_t old_word;
IGNORE_RACES_START ();
BLOCK_CANCELLATIONS; /* not supported yet */
BLOCK_CANCELLATIONS;
old_word = ATM_LOAD (&mu->word);
if ((old_word & MU_ANY_LOCK) == 0) {
nsync_panic_ ("nsync_mu not held in some mode when calling "
@ -165,12 +165,12 @@ int nsync_mu_wait_with_deadline (nsync_mu *mu,
l_type = nsync_reader_type_;
}
w->tag = 0; /* avoid allocating system resources */
first_wait = 1; /* first time through the loop below. */
condition_is_true = (condition == NULL || (*condition) (condition_arg));
/* Loop until either the condition becomes true, or "outcome" indicates
cancellation or timeout. */
w = NULL;
outcome = 0;
while (outcome == 0 && !condition_is_true) {
uint32_t has_condition;
@ -180,8 +180,10 @@ int nsync_mu_wait_with_deadline (nsync_mu *mu,
int sem_outcome;
unsigned attempts;
int have_lock;
if (w == NULL) {
w = nsync_waiter_new_ (); /* get a waiter struct if we need one. */
/* initialize the waiter if we haven't already */
if (!w->tag) {
nsync_waiter_init_ (w);
}
/* Prepare to wait. */
@ -259,9 +261,7 @@ int nsync_mu_wait_with_deadline (nsync_mu *mu,
}
condition_is_true = (condition == NULL || (*condition) (condition_arg));
}
if (w != NULL) {
nsync_waiter_free_ (w); /* free waiter if we allocated one. */
}
nsync_waiter_destroy_ (w);
if (condition_is_true) {
outcome = 0; /* condition is true trumps other outcomes. */
}
@ -319,5 +319,3 @@ void nsync_mu_unlock_without_wakeup (nsync_mu *mu) {
}
IGNORE_RACES_END ();
}

10
third_party/nsync/mem/nsync_wait.c vendored
View file

@ -17,6 +17,7 @@
╚─────────────────────────────────────────────────────────────────────────────*/
#include "libc/calls/blockcancel.internal.h"
#include "libc/mem/mem.h"
#include "libc/thread/thread.h"
#include "third_party/nsync/atomic.h"
#include "third_party/nsync/atomic.internal.h"
#include "third_party/nsync/common.internal.h"
@ -36,7 +37,7 @@ int nsync_wait_n (void *mu, void (*lock) (void *), void (*unlock) (void *),
int count, struct nsync_waitable_s *waitable[]) {
int ready;
IGNORE_RACES_START ();
BLOCK_CANCELLATIONS; /* TODO(jart): Does this need pthread cancellations? */
BLOCK_CANCELLATIONS;
for (ready = 0; ready != count &&
nsync_time_cmp ((*waitable[ready]->funcs->ready_time) (
waitable[ready]->v, NULL),
@ -48,7 +49,8 @@ int nsync_wait_n (void *mu, void (*lock) (void *), void (*unlock) (void *),
int unlocked = 0;
int j;
int enqueued = 1;
waiter *w = nsync_waiter_new_ ();
waiter w[1];
nsync_waiter_init_ (w);
struct nsync_waiter_s nw_set[4];
struct nsync_waiter_s *nw = nw_set;
if (count > (int) (sizeof (nw_set) / sizeof (nw_set[0]))) {
@ -95,10 +97,10 @@ int nsync_wait_n (void *mu, void (*lock) (void *), void (*unlock) (void *),
}
}
nsync_waiter_destroy_ (w);
if (nw != nw_set) {
free (nw);
}
nsync_waiter_free_ (w);
if (unlocked) {
(*lock) (mu);
}
@ -107,5 +109,3 @@ int nsync_wait_n (void *mu, void (*lock) (void *), void (*unlock) (void *),
IGNORE_RACES_END ();
return (ready);
}

17
third_party/nsync/mu.c vendored
View file

@ -103,7 +103,8 @@ void nsync_mu_lock_slow_ (nsync_mu *mu, waiter *w, uint32_t clear, lock_type *l_
/* wait until awoken. */
while (ATM_LOAD_ACQ (&w->nw.waiting) != 0) { /* acquire load */
nsync_mu_semaphore_p (&w->sem);
/* This can only return 0 or ECANCELED. */
ASSERT (nsync_mu_semaphore_p (&w->sem) == 0);
}
wait_count++;
/* If the thread has been woken more than this many
@ -155,9 +156,10 @@ void nsync_mu_lock (nsync_mu *mu) {
if ((old_word&MU_WZERO_TO_ACQUIRE) != 0 ||
!ATM_CAS_ACQ (&mu->word, old_word,
(old_word+MU_WADD_TO_ACQUIRE) & ~MU_WCLEAR_ON_ACQUIRE)) {
waiter *w = nsync_waiter_new_ ();
nsync_mu_lock_slow_ (mu, w, 0, nsync_writer_type_);
nsync_waiter_free_ (w);
waiter w;
nsync_waiter_init_ (&w);
nsync_mu_lock_slow_ (mu, &w, 0, nsync_writer_type_);
nsync_waiter_destroy_ (&w);
}
}
IGNORE_RACES_END ();
@ -190,9 +192,10 @@ void nsync_mu_rlock (nsync_mu *mu) {
if ((old_word&MU_RZERO_TO_ACQUIRE) != 0 ||
!ATM_CAS_ACQ (&mu->word, old_word,
(old_word+MU_RADD_TO_ACQUIRE) & ~MU_RCLEAR_ON_ACQUIRE)) {
waiter *w = nsync_waiter_new_ ();
nsync_mu_lock_slow_ (mu, w, 0, nsync_reader_type_);
nsync_waiter_free_ (w);
waiter w;
nsync_waiter_init_ (&w);
nsync_mu_lock_slow_ (mu, &w, 0, nsync_reader_type_);
nsync_waiter_destroy_ (&w);
}
}
IGNORE_RACES_END ();

30
third_party/nsync/mu_semaphore.c vendored
View file

@ -16,6 +16,7 @@
│ limitations under the License. │
╚─────────────────────────────────────────────────────────────────────────────*/
#include "third_party/nsync/mu_semaphore.h"
#include "libc/calls/cp.internal.h"
#include "libc/dce.h"
#include "third_party/nsync/mu_semaphore.internal.h"
@ -36,28 +37,45 @@ void nsync_mu_semaphore_init (nsync_semaphore *s) {
}
}
/* Releases system resources associated with *s. */
void nsync_mu_semaphore_destroy (nsync_semaphore *s) {
if (IsXnuSilicon ()) {
return nsync_mu_semaphore_destroy_gcd (s);
} else if (IsNetbsd ()) {
return nsync_mu_semaphore_destroy_sem (s);
}
}
/* Wait until the count of *s exceeds 0, and decrement it. */
errno_t nsync_mu_semaphore_p (nsync_semaphore *s) {
errno_t err;
BEGIN_CANCELLATION_POINT;
if (IsXnuSilicon ()) {
return nsync_mu_semaphore_p_gcd (s);
err = nsync_mu_semaphore_p_gcd (s);
} else if (IsNetbsd ()) {
return nsync_mu_semaphore_p_sem (s);
err = nsync_mu_semaphore_p_sem (s);
} else {
return nsync_mu_semaphore_p_futex (s);
err = nsync_mu_semaphore_p_futex (s);
}
END_CANCELLATION_POINT;
return err;
}
/* Wait until one of:
the count of *s is non-zero, in which case decrement *s and return 0;
or abs_deadline expires, in which case return ETIMEDOUT. */
errno_t nsync_mu_semaphore_p_with_deadline (nsync_semaphore *s, nsync_time abs_deadline) {
errno_t err;
BEGIN_CANCELLATION_POINT;
if (IsXnuSilicon ()) {
return nsync_mu_semaphore_p_with_deadline_gcd (s, abs_deadline);
err = nsync_mu_semaphore_p_with_deadline_gcd (s, abs_deadline);
} else if (IsNetbsd ()) {
return nsync_mu_semaphore_p_with_deadline_sem (s, abs_deadline);
err = nsync_mu_semaphore_p_with_deadline_sem (s, abs_deadline);
} else {
return nsync_mu_semaphore_p_with_deadline_futex (s, abs_deadline);
err = nsync_mu_semaphore_p_with_deadline_futex (s, abs_deadline);
}
END_CANCELLATION_POINT;
return err;
}
/* Ensure that the count of *s is at least 1. */

3
third_party/nsync/mu_semaphore.h vendored
View file

@ -11,6 +11,9 @@ typedef struct nsync_semaphore_s_ {
/* Initialize *s; the initial value is 0. */
void nsync_mu_semaphore_init(nsync_semaphore *s);
/* Releases system resources associated with *s. */
void nsync_mu_semaphore_destroy(nsync_semaphore *s);
/* Wait until the count of *s exceeds 0, and decrement it. */
errno_t nsync_mu_semaphore_p(nsync_semaphore *s);

2
third_party/nsync/mu_semaphore.internal.h vendored
View file

@ -11,11 +11,13 @@ errno_t nsync_mu_semaphore_p_with_deadline_futex(nsync_semaphore *, nsync_time);
void nsync_mu_semaphore_v_futex(nsync_semaphore *);
void nsync_mu_semaphore_init_sem(nsync_semaphore *);
void nsync_mu_semaphore_destroy_sem(nsync_semaphore *);
errno_t nsync_mu_semaphore_p_sem(nsync_semaphore *);
errno_t nsync_mu_semaphore_p_with_deadline_sem(nsync_semaphore *, nsync_time);
void nsync_mu_semaphore_v_sem(nsync_semaphore *);
void nsync_mu_semaphore_init_gcd(nsync_semaphore *);
void nsync_mu_semaphore_destroy_gcd(nsync_semaphore *);
errno_t nsync_mu_semaphore_p_gcd(nsync_semaphore *);
errno_t nsync_mu_semaphore_p_with_deadline_gcd(nsync_semaphore *, nsync_time);
void nsync_mu_semaphore_v_gcd(nsync_semaphore *);

10
third_party/nsync/mu_semaphore_gcd.c vendored
View file

@ -41,6 +41,11 @@ static dispatch_semaphore_t dispatch_semaphore_create(long count) {
return (ds);
}
static void dispatch_release (dispatch_semaphore_t ds) {
__syslib->dispatch_release (ds);
STRACE ("dispatch_release(%#lx)", ds);
}
static long dispatch_semaphore_wait (dispatch_semaphore_t ds,
dispatch_time_t dt) {
long rc = __syslib->dispatch_semaphore_wait (ds, dt);
@ -65,6 +70,11 @@ void nsync_mu_semaphore_init_gcd (nsync_semaphore *s) {
*(dispatch_semaphore_t *)s = dispatch_semaphore_create (0);
}
/* Releases system resources associated with *s. */
void nsync_mu_semaphore_destroy_gcd (nsync_semaphore *s) {
dispatch_release (*(dispatch_semaphore_t *)s);
}
/* Wait until the count of *s exceeds 0, and decrement it. */
errno_t nsync_mu_semaphore_p_gcd (nsync_semaphore *s) {
dispatch_semaphore_wait (*(dispatch_semaphore_t *)s,

22
third_party/nsync/mu_semaphore_sem.c vendored
View file

@ -1,5 +1,5 @@
/*-*- mode:c;indent-tabs-mode:nil;c-basic-offset:2;tab-width:8;coding:utf-8 -*-│
│vi: set net ft=c ts=2 sts=2 sw=2 fenc=utf-8 :vi│
/*-*- mode:c;indent-tabs-mode:t;c-basic-offset:8;tab-width:8;coding:utf-8 -*-│
│vi: set et ft=c ts=8 tw=8 fenc=utf-8 :vi│
╞══════════════════════════════════════════════════════════════════════════════╡
│ Copyright 2022 Justine Alexandra Roberts Tunney │
│ │
@ -25,6 +25,7 @@
#include "libc/intrin/strace.internal.h"
#include "libc/str/str.h"
#include "libc/sysv/consts/f.h"
#include "libc/sysv/consts/fd.h"
#include "third_party/nsync/mu_semaphore.h"
#include "third_party/nsync/time.h"
// clang-format off
@ -44,14 +45,19 @@ static nsync_semaphore *sem_big_enough_for_sem = (nsync_semaphore *) (uintptr_t)
/* Initialize *s; the initial value is 0. */
void nsync_mu_semaphore_init_sem (nsync_semaphore *s) {
int newfd;
struct sem *f = (struct sem *) s;
f->id = 0;
ASSERT (!sys_sem_init (0, &f->id));
STRACE ("sem_init(0, [%ld]) → 0", f->id);
ASSERT ((newfd = __sys_fcntl (f->id, F_DUPFD_CLOEXEC, 100)) != -1);
ASSERT (!sys_sem_destroy (f->id));
f->id = newfd;
ASSERT (__sys_fcntl (f->id, F_SETFD, FD_CLOEXEC) == 0); // ouch
}
/* Releases system resources associated with *s. */
void nsync_mu_semaphore_destroy_sem (nsync_semaphore *s) {
int rc;
struct sem *f = (struct sem *) s;
ASSERT (!(rc = sys_sem_destroy (f->id)));
STRACE ("sem_destroy(%ld) → %d", rc);
}
/* Wait until the count of *s exceeds 0, and decrement it. */
@ -60,9 +66,7 @@ errno_t nsync_mu_semaphore_p_sem (nsync_semaphore *s) {
errno_t result;
struct sem *f = (struct sem *) s;
e = errno;
BEGIN_CANCELLATION_POINT;
rc = sys_sem_wait (f->id);
END_CANCELLATION_POINT;
STRACE ("sem_wait(%ld) → %d% m", f->id, rc);
if (!rc) {
result = 0;
@ -82,9 +86,7 @@ errno_t nsync_mu_semaphore_p_with_deadline_sem (nsync_semaphore *s, nsync_time a
errno_t result;
struct sem *f = (struct sem *) s;
e = errno;
BEGIN_CANCELLATION_POINT;
rc = sys_sem_timedwait (f->id, &abs_deadline);
END_CANCELLATION_POINT;
STRACE ("sem_timedwait(%ld, %s) → %d% m", f->id,
DescribeTimespec(0, &abs_deadline), rc);
if (!rc) {

2
third_party/nsync/testing/cv_test.c vendored
View file

@ -17,9 +17,9 @@
╚─────────────────────────────────────────────────────────────────────────────*/
#include "third_party/nsync/cv.h"
#include "libc/errno.h"
#include "libc/fmt/fmt.h"
#include "libc/mem/mem.h"
#include "libc/runtime/runtime.h"
#include "libc/stdio/stdio.h"
#include "libc/str/str.h"
#include "third_party/nsync/debug.h"
#include "third_party/nsync/mu.h"

2
third_party/nsync/testing/cv_wait_example_test.c vendored
View file

@ -15,7 +15,7 @@
│ See the License for the specific language governing permissions and │
│ limitations under the License. │
╚─────────────────────────────────────────────────────────────────────────────*/
#include "libc/fmt/fmt.h"
#include "libc/stdio/stdio.h"
#include "libc/str/str.h"
#include "third_party/nsync/array.internal.h"
#include "third_party/nsync/cv.h"

3
third_party/nsync/testing/dll_test.c vendored
View file

@ -16,9 +16,8 @@
│ limitations under the License. │
╚─────────────────────────────────────────────────────────────────────────────*/
#include "libc/intrin/dll.h"
#include "libc/fmt/fmt.h"
#include "libc/intrin/dll.h"
#include "libc/mem/mem.h"
#include "libc/stdio/stdio.h"
#include "libc/str/str.h"
#include "third_party/nsync/array.internal.h"
#include "third_party/nsync/testing/smprintf.h"

2
third_party/nsync/testing/mu_wait_example_test.c vendored
View file

@ -15,7 +15,7 @@
│ See the License for the specific language governing permissions and │
│ limitations under the License. │
╚─────────────────────────────────────────────────────────────────────────────*/
#include "libc/fmt/fmt.h"
#include "libc/stdio/stdio.h"
#include "libc/str/str.h"
#include "third_party/nsync/array.internal.h"
#include "third_party/nsync/heap.internal.h"

6
third_party/nsync/testing/smprintf.c vendored
View file

@ -15,10 +15,10 @@
│ See the License for the specific language governing permissions and │
│ limitations under the License. │
╚─────────────────────────────────────────────────────────────────────────────*/
#include "libc/fmt/fmt.h"
#include "libc/mem/mem.h"
#include "libc/str/str.h"
#include "third_party/nsync/testing/smprintf.h"
#include "libc/mem/mem.h"
#include "libc/stdio/stdio.h"
#include "libc/str/str.h"
// clang-format off
char *smprintf (const char *fmt, ...) {

3
third_party/python/Modules/cjkcodecs/xloadzd.c vendored
View file

@ -19,7 +19,6 @@
#include "libc/assert.h"
#include "libc/fmt/leb128.h"
#include "libc/intrin/atomic.h"
#include "libc/intrin/kmalloc.h"
#include "libc/mem/mem.h"
#include "libc/nexgen32e/crc32.h"
#include "libc/runtime/internal.h"
@ -53,7 +52,7 @@ void *xloadzd(_Atomic(void *) *a, const void *p, size_t n, size_t m, size_t c,
free(q);
return 0;
}
if (!(r = kmalloc(c * z))) {
if (!(r = malloc(c * z))) {
free(q);
return 0;
}

26
tool/build/runitd.c
View file

@ -264,9 +264,8 @@ void StartTcpServer(void) {
INFOF("listening on tcp:%s", DescribeAddress(&g_servaddr));
if (g_sendready) {
printf("ready %hu\n", ntohs(g_servaddr.sin_port));
fflush(stdout);
fclose(stdout);
dup2(g_bogusfd, fileno(stdout));
close(1);
dup2(g_bogusfd, 1);
}
}
@ -698,6 +697,7 @@ WaitAgain:
}
void HandleClient(void) {
struct stat st;
struct Client *client;
client = calloc(1, sizeof(struct Client));
client->addrsize = sizeof(client->addr);
@ -720,6 +720,9 @@ void HandleClient(void) {
WARNF("poll failed %m");
}
}
if (fstat(2, &st) != -1 && st.st_size > kLogMaxBytes) {
ftruncate(2, 0); // auto rotate log
}
sigset_t mask;
pthread_attr_t attr;
sigfillset(&mask);
@ -749,17 +752,18 @@ int Serve(void) {
}
void Daemonize(void) {
VERBF("Daemonize");
struct stat st;
if (fork() > 0) _exit(0);
setsid();
if (fork() > 0) _exit(0);
dup2(g_bogusfd, fileno(stdin));
if (!g_sendready) dup2(g_bogusfd, fileno(stdout));
freopen(kLogFile, "ae", stderr);
if (fstat(fileno(stderr), &st) != -1 && st.st_size > kLogMaxBytes) {
ftruncate(fileno(stderr), 0);
dup2(g_bogusfd, 0);
if (!g_sendready) dup2(g_bogusfd, 1);
close(2);
open(kLogFile, O_CREAT | O_WRONLY | O_APPEND | O_CLOEXEC, 0644);
extern long __klog_handle;
if (__klog_handle > 0) {
close(__klog_handle);
}
__klog_handle = 2;
}
int main(int argc, char *argv[]) {
@ -779,8 +783,8 @@ int main(int argc, char *argv[]) {
} else {
g_bogusfd = open("/dev/zero", O_RDONLY | O_CLOEXEC);
}
if (g_daemonize) Daemonize();
mkdir("o", 0700);
if (g_daemonize) Daemonize();
Serve();
free(g_psk);
#if IsModeDbg()