mirrors/cosmopolitan
1
0
Fork 0
mirror of https://github.com/jart/cosmopolitan.git synced 2025-02-07 06:53:33 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

Make more threading improvements

Browse source 
- ASAN memory morgue is now lockless
- Make C11 atomics header more portable
- Rewrote pthread keys support to be lockless
- Simplify Python's unicode table unpacking code
- Make crash report write(2) closer to being atomic
- Make it possible to strace/ftrace a single thread
- ASAN now checks nul-terminated strings fast and properly
- Windows fork() now restores TLS memory of calling thread
This commit is contained in:
Justine Tunney 2022-11-01 22:36:03 -07:00
parent d7b88734cd
commit e522aa3a07
No known key found for this signature in database
GPG key ID: BE714B4575D6E328
189 changed files with 1363 additions and 1217 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

5
libc/calls/__sig_mask.c
View file

@ -47,7 +47,10 @@ textwindows int __sig_mask(int how, const sigset_t *neu, sigset_t *old) {
} else {
x = y;
}
x &= ~(GetSigBit(SIGKILL) | GetSigBit(SIGSTOP) | GetSigBit(SIGABRT));
x &= ~(0
#define M(x) | GetSigBit(x)
#include "libc/intrin/sigisprecious.inc"
);
*mask = x;
}
return 0;

2
libc/calls/chdir.c
View file

@ -36,7 +36,7 @@
int chdir(const char *path) {
int rc;
GetProgramExecutableName(); // XXX: ugly workaround
if (!path || (IsAsan() && !__asan_is_valid(path, 1))) {
if (!path || (IsAsan() && !__asan_is_valid_str(path))) {
rc = efault();
} else if (!IsWindows()) {
rc = sys_chdir(path);

2
libc/calls/chroot.c
View file

@ -49,7 +49,7 @@
*/
int chroot(const char *path) {
int rc;
if (!path || (IsAsan() && !__asan_is_valid(path, 1))) {
if (!path || (IsAsan() && !__asan_is_valid_str(path))) {
rc = efault();
} else {
rc = sys_chroot(path);

2
libc/calls/execve.c
View file

@ -54,7 +54,7 @@ int execve(const char *prog, char *const argv[], char *const envp[]) {
int rc;
size_t i;
if (!prog || !argv || !envp ||
(IsAsan() && (!__asan_is_valid(prog, 1) || //
(IsAsan() && (!__asan_is_valid_str(prog) || //
!__asan_is_valid_strlist(argv) || //
!__asan_is_valid_strlist(envp)))) {
rc = efault();

2
libc/calls/execvpe.c
View file

@ -37,7 +37,7 @@
int execvpe(const char *prog, char *const argv[], char *const *envp) {
char *exe;
char pathbuf[PATH_MAX];
if (IsAsan() && !__asan_is_valid(prog, 1)) return efault();
if (IsAsan() && !__asan_is_valid_str(prog)) return efault();
if (!(exe = commandv(prog, pathbuf, sizeof(pathbuf)))) return -1;
return execve(exe, argv, envp);
}

2
libc/calls/faccessat.c
View file

@ -51,7 +51,7 @@
int faccessat(int dirfd, const char *path, int amode, int flags) {
int e, rc;
struct ZiposUri zipname;
if (!path || (IsAsan() && !__asan_is_valid(path, 1))) {
if (!path || (IsAsan() && !__asan_is_valid_str(path))) {
rc = efault();
} else if (__isfdkind(dirfd, kFdZip)) {
rc = enotsup();

2
libc/calls/fchmodat.c
View file

@ -44,7 +44,7 @@
*/
int fchmodat(int dirfd, const char *path, uint32_t mode, int flags) {
int rc;
if (IsAsan() && !__asan_is_valid(path, 1)) {
if (IsAsan() && !__asan_is_valid_str(path)) {
rc = efault();
} else if (_weaken(__zipos_notat) &&
(rc = __zipos_notat(dirfd, path)) == -1) {

2
libc/calls/fchownat.c
View file

@ -43,7 +43,7 @@
int fchownat(int dirfd, const char *path, uint32_t uid, uint32_t gid,
int flags) {
int rc;
if (IsAsan() && !__asan_is_valid(path, 1)) {
if (IsAsan() && !__asan_is_valid_str(path)) {
rc = efault();
} else if (_weaken(__zipos_notat) &&
(rc = __zipos_notat(dirfd, path)) == -1) {

20
libc/calls/fexecve.c
View file

@ -21,8 +21,7 @@
#include "libc/dce.h"
#include "libc/fmt/itoa.h"
#include "libc/intrin/asan.internal.h"
#include "libc/intrin/kprintf.h"
#include "libc/intrin/likely.h"
#include "libc/intrin/describeflags.internal.h"
#include "libc/intrin/strace.internal.h"
#include "libc/str/str.h"
#include "libc/sysv/errfuns.h"
@ -48,21 +47,8 @@ int fexecve(int fd, char *const argv[], char *const envp[]) {
(!__asan_is_valid_strlist(argv) || !__asan_is_valid_strlist(envp)))) {
rc = efault();
} else {
#ifdef SYSDEBUG
if (UNLIKELY(__strace > 0)) {
kprintf(STRACE_PROLOGUE "fexecve(%d, {", fd);
for (i = 0; argv[i]; ++i) {
if (i) kprintf(", ");
kprintf("%#s", argv[i]);
}
kprintf("}, {");
for (i = 0; envp[i]; ++i) {
if (i) kprintf(", ");
kprintf("%#s", envp[i]);
}
kprintf("})\n");
}
#endif
STRACE("fexecve(%d, %s, %s) → ...", fd, DescribeStringList(argv),
DescribeStringList(envp));
if (IsLinux()) {
char path[14 + 12];
FormatInt32(stpcpy(path, "/proc/self/fd/"), fd);

2
libc/calls/fileexists.c
View file

@ -51,7 +51,7 @@ bool fileexists(const char *path) {
struct ZiposUri zipname;
uint16_t path16[PATH_MAX];
e = errno;
if (IsAsan() && !__asan_is_valid(path, 1)) {
if (IsAsan() && !__asan_is_valid_str(path)) {
efault();
res = false;
} else if (_weaken(__zipos_open) &&

2
libc/calls/fstatat-sysv.c
View file

@ -31,7 +31,7 @@ int32_t sys_fstatat(int32_t dirfd, const char *path, struct stat *st,
int rc;
void *p;
union metastat ms;
if (IsAsan() && !__asan_is_valid(path, 1)) return efault();
if (IsAsan() && !__asan_is_valid_str(path)) return efault();
if (IsLinux()) {
_Static_assert(sizeof(*st) == sizeof(ms.linux), "assumption broken");
if (IsAsan() && (st && !__asan_is_valid(st, sizeof(*st)))) return efault();

2
libc/calls/isdirectory.c
View file

@ -51,7 +51,7 @@ bool isdirectory(const char *path) {
union metastat st;
struct ZiposUri zipname;
e = errno;
if (IsAsan() && !__asan_is_valid(path, 1)) {
if (IsAsan() && !__asan_is_valid_str(path)) {
efault();
res = false;
} else if (_weaken(__zipos_open) &&

2
libc/calls/isregularfile.c
View file

@ -48,7 +48,7 @@ bool isregularfile(const char *path) {
union metastat st;
struct ZiposUri zipname;
e = errno;
if (IsAsan() && !__asan_is_valid(path, 1)) {
if (IsAsan() && !__asan_is_valid_str(path)) {
efault();
res = false;
} else if (_weaken(__zipos_open) &&

2
libc/calls/issymlink.c
View file

@ -51,7 +51,7 @@ bool issymlink(const char *path) {
union metastat st;
struct ZiposUri zipname;
e = errno;
if (IsAsan() && !__asan_is_valid(path, 1)) {
if (IsAsan() && !__asan_is_valid_str(path)) {
efault();
res = false;
} else if (_weaken(__zipos_open) &&

2
libc/calls/linkat.c
View file

@ -41,7 +41,7 @@ int linkat(int olddirfd, const char *oldpath, int newdirfd, const char *newpath,
int flags) {
int rc;
if (IsAsan() &&
(!__asan_is_valid(oldpath, 1) || !__asan_is_valid(newpath, 1))) {
(!__asan_is_valid_str(oldpath) || !__asan_is_valid_str(newpath))) {
rc = efault();
} else if (_weaken(__zipos_notat) &&
((rc = __zipos_notat(olddirfd, oldpath)) == -1 ||

2
libc/calls/mkdirat.c
View file

@ -53,7 +53,7 @@
*/
int mkdirat(int dirfd, const char *path, unsigned mode) {
int rc;
if (IsAsan() && !__asan_is_valid(path, 1)) {
if (IsAsan() && !__asan_is_valid_str(path)) {
rc = efault();
} else if (_weaken(__zipos_notat) &&
(rc = __zipos_notat(dirfd, path)) == -1) {

4
libc/calls/mkfifo.c
View file

@ -17,10 +17,10 @@
│ PERFORMANCE OF THIS SOFTWARE. │
╚─────────────────────────────────────────────────────────────────────────────*/
#include "libc/calls/calls.h"
#include "libc/intrin/strace.internal.h"
#include "libc/calls/syscall-sysv.internal.h"
#include "libc/dce.h"
#include "libc/intrin/asan.internal.h"
#include "libc/intrin/strace.internal.h"
#include "libc/nt/ipc.h"
#include "libc/sysv/consts/s.h"
#include "libc/sysv/errfuns.h"
@ -36,7 +36,7 @@
int mkfifo(const char *pathname, unsigned mode) {
// TODO(jart): Windows?
int rc;
if (IsAsan() && !__asan_is_valid(pathname, 1)) {
if (IsAsan() && !__asan_is_valid_str(pathname)) {
rc = efault();
} else if (IsLinux()) {
rc = sys_mknod(pathname, mode | S_IFIFO, 0);

4
libc/calls/mknod.c
View file

@ -17,10 +17,10 @@
│ PERFORMANCE OF THIS SOFTWARE. │
╚─────────────────────────────────────────────────────────────────────────────*/
#include "libc/calls/calls.h"
#include "libc/intrin/strace.internal.h"
#include "libc/calls/syscall-sysv.internal.h"
#include "libc/dce.h"
#include "libc/intrin/asan.internal.h"
#include "libc/intrin/strace.internal.h"
#include "libc/sysv/consts/s.h"
#include "libc/sysv/errfuns.h"
@ -40,7 +40,7 @@
*/
int mknod(const char *path, uint32_t mode, uint64_t dev) {
int rc;
if (IsAsan() && !__asan_is_valid(path, 1)) return efault();
if (IsAsan() && !__asan_is_valid_str(path)) return efault();
if (mode & S_IFREG) return creat(path, mode & ~S_IFREG);
if (mode & S_IFDIR) return mkdir(path, mode & ~S_IFDIR);
if (mode & S_IFIFO) return mkfifo(path, mode & ~S_IFIFO);

12
libc/calls/mkntpath.c
View file

@ -21,9 +21,7 @@
#include "libc/intrin/strace.internal.h"
#include "libc/macros.internal.h"
#include "libc/nt/systeminfo.h"
#include "libc/str/oldutf16.internal.h"
#include "libc/str/str.h"
#include "libc/str/tpdecode.internal.h"
#include "libc/sysv/consts/o.h"
#include "libc/sysv/errfuns.h"
@ -61,9 +59,13 @@ textwindows int __mkntpath(const char *path,
/**
* Copies path for Windows NT.
*
* This entails (1) UTF-8 to UTF-16 conversion; (2) replacing
* forward-slashes with backslashes; and (3) remapping several
* well-known paths (e.g. /dev/null → NUL) for convenience.
* This function does the following chores:
*
* 1. Converting UTF-8 to UTF-16
* 2. Replacing forward-slashes with backslashes
* 3. Fixing drive letter paths, e.g. `/c/` → `c:\`
* 4. Add `\\?\` prefix for paths exceeding 260 chars
* 5. Remapping well-known paths, e.g. `/dev/null` → `NUL`
*
* @param flags is used by open()
* @param path16 is shortened so caller can prefix, e.g. \\.\pipe\, and

2
libc/calls/munmap-metal.c
View file

@ -16,7 +16,7 @@
│ TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR │
│ PERFORMANCE OF THIS SOFTWARE. │
╚─────────────────────────────────────────────────────────────────────────────*/
#include "libc/runtime/directmap.internal.h"
#include "libc/intrin/directmap.internal.h"
#include "libc/runtime/pc.internal.h"
noasan int sys_munmap_metal(void *addr, size_t size) {

6
libc/calls/munmap-sysv.c
View file

@ -16,11 +16,11 @@
│ TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR │
│ PERFORMANCE OF THIS SOFTWARE. │
╚─────────────────────────────────────────────────────────────────────────────*/
#include "libc/intrin/strace.internal.h"
#include "libc/calls/syscall-sysv.internal.h"
#include "libc/dce.h"
#include "libc/intrin/describeflags.internal.h"
#include "libc/runtime/directmap.internal.h"
#include "libc/runtime/memtrack.internal.h"
#include "libc/intrin/directmap.internal.h"
#include "libc/intrin/strace.internal.h"
/**
* Unmaps memory directly with system.

4
libc/calls/ntspawn.c
View file

@ -18,9 +18,7 @@
╚─────────────────────────────────────────────────────────────────────────────*/
#include "libc/calls/ntspawn.h"
#include "libc/calls/syscall_support-nt.internal.h"
#include "libc/intrin/kprintf.h"
#include "libc/intrin/pushpop.h"
#include "libc/intrin/strace.internal.h"
#include "libc/macros.internal.h"
#include "libc/nt/enum/filemapflags.h"
#include "libc/nt/enum/pageflags.h"
@ -80,6 +78,8 @@ textwindows int ntspawn(
rc = -1;
block = NULL;
if (__mkntpath(prog, prog16) == -1) return -1;
// we can't call malloc() because we're higher in the topological order
// we can't call kmalloc() because fork() calls this when kmalloc is locked
if ((handle = CreateFileMapping(-1, 0, pushpop(kNtPageReadwrite), 0,
sizeof(*block), 0)) &&
(block = MapViewOfFileEx(handle, kNtFileMapRead | kNtFileMapWrite, 0, 0,

2
libc/calls/openat.c
View file

@ -154,7 +154,7 @@ int openat(int dirfd, const char *file, int flags, ...) {
va_start(va, flags);
mode = va_arg(va, unsigned);
va_end(va);
if (file && (!IsAsan() || __asan_is_valid(file, 1))) {
if (file && (!IsAsan() || __asan_is_valid_str(file))) {
if (!__isfdkind(dirfd, kFdZip)) {
if (_weaken(__zipos_open) &&
_weaken(__zipos_parseuri)(file, &zipname) != -1) {

31
libc/calls/poll.c
View file

@ -16,17 +16,9 @@
│ TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR │
│ PERFORMANCE OF THIS SOFTWARE. │
╚─────────────────────────────────────────────────────────────────────────────*/
#include "libc/calls/calls.h"
#include "libc/dce.h"
#include "libc/errno.h"
#include "libc/intrin/asan.internal.h"
#include "libc/intrin/describeflags.internal.h"
#include "libc/intrin/kprintf.h"
#include "libc/intrin/likely.h"
#include "libc/intrin/strace.internal.h"
#include "libc/macros.internal.h"
#include "libc/sock/internal.h"
#include "libc/sock/sock.h"
#include "libc/sock/struct/pollfd.h"
#include "libc/sock/struct/pollfd.internal.h"
#include "libc/sysv/errfuns.h"
@ -68,8 +60,8 @@
* @norestart
*/
int poll(struct pollfd *fds, size_t nfds, int timeout_ms) {
int rc;
size_t n;
int i, rc;
uint64_t millis;
if (IsAsan() && (__builtin_mul_overflow(nfds, sizeof(struct pollfd), &n) ||
@ -86,24 +78,7 @@ int poll(struct pollfd *fds, size_t nfds, int timeout_ms) {
rc = sys_poll_nt(fds, nfds, &millis, 0);
}
#if defined(SYSDEBUG) && _POLLTRACE
if (UNLIKELY(__strace > 0)) {
kprintf(STRACE_PROLOGUE "poll(");
if ((!IsAsan() && kisdangerous(fds)) ||
(IsAsan() && !__asan_is_valid(fds, nfds * sizeof(struct pollfd)))) {
kprintf("%p", fds);
} else {
kprintf("[{");
for (i = 0; i < MIN(5, nfds); ++i) {
kprintf("%s{%d, %s, %s}", i ? ", " : "", fds[i].fd,
DescribePollFlags(fds[i].events),
DescribePollFlags(fds[i].revents));
}
kprintf("%s}]", i == 5 ? "..." : "");
}
kprintf(", %'zu, %'d) → %d% lm\n", nfds, timeout_ms, rc);
}
#endif
STRACE("poll(%s, %'zu, %'d) → %d% lm\n", DescribePollFds(rc, fds, nfds), nfds,
timeout_ms, rc);
return rc;
}

29
libc/calls/ppoll.c
View file

@ -16,15 +16,14 @@
│ TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR │
│ PERFORMANCE OF THIS SOFTWARE. │
╚─────────────────────────────────────────────────────────────────────────────*/
#include "libc/calls/calls.h"
#include "libc/calls/struct/sigset.h"
#include "libc/calls/struct/sigset.internal.h"
#include "libc/calls/struct/timespec.h"
#include "libc/calls/struct/timespec.internal.h"
#include "libc/dce.h"
#include "libc/errno.h"
#include "libc/intrin/asan.internal.h"
#include "libc/intrin/kprintf.h"
#include "libc/intrin/strace.internal.h"
#include "libc/macros.internal.h"
#include "libc/sock/struct/pollfd.h"
#include "libc/sock/struct/pollfd.internal.h"
#include "libc/sysv/consts/sig.h"
@ -58,7 +57,7 @@
int ppoll(struct pollfd *fds, size_t nfds, const struct timespec *timeout,
const sigset_t *sigmask) {
size_t n;
int e, i, rc;
int e, rc;
uint64_t millis;
sigset_t oldmask;
struct timespec ts, *tsp;
@ -96,25 +95,7 @@ int ppoll(struct pollfd *fds, size_t nfds, const struct timespec *timeout,
rc = sys_poll_nt(fds, nfds, &millis, sigmask);
}
#if defined(SYSDEBUG) && _POLLTRACE
if (UNLIKELY(__strace > 0)) {
kprintf(STRACE_PROLOGUE "ppoll(");
if ((!IsAsan() && kisdangerous(fds)) ||
(IsAsan() && !__asan_is_valid(fds, nfds * sizeof(struct pollfd)))) {
kprintf("%p", fds);
} else {
kprintf("[{");
for (i = 0; i < MIN(5, nfds); ++i) {
kprintf("%s{%d, %s, %s}", i ? ", " : "", fds[i].fd,
DescribePollFlags(fds[i].events),
DescribePollFlags(fds[i].revents));
}
kprintf("%s}]", i == 5 ? "..." : "");
}
kprintf(", %'zu, %s, %s) → %d% lm\n", nfds, DescribeTimeval(0, timeout),
DescribeSigset(0, sigmask), rc);
}
#endif
STRACE("ppoll(%s, %'zu, %s, %s) → %d% lm\n", DescribePollFds(rc, fds, nfds),
nfds, DescribeTimespec(0, timeout), DescribeSigset(0, sigmask), rc);
return rc;
}

2
libc/calls/renameat.c
View file

@ -47,7 +47,7 @@ int renameat(int olddirfd, const char *oldpath, int newdirfd,
const char *newpath) {
int rc;
if (IsAsan() &&
(!__asan_is_valid(oldpath, 1) || !__asan_is_valid(newpath, 1))) {
(!__asan_is_valid_str(oldpath) || !__asan_is_valid_str(newpath))) {
rc = efault();
} else if (_weaken(__zipos_notat) &&
((rc = __zipos_notat(olddirfd, oldpath)) == -1 ||

12
libc/calls/reservefd.c
View file

@ -16,26 +16,16 @@
│ TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR │
│ PERFORMANCE OF THIS SOFTWARE. │
╚─────────────────────────────────────────────────────────────────────────────*/
#include "libc/assert.h"
#include "libc/calls/calls.h"
#include "libc/calls/internal.h"
#include "libc/calls/state.internal.h"
#include "libc/calls/struct/sigset.h"
#include "libc/dce.h"
#include "libc/calls/struct/fd.internal.h"
#include "libc/intrin/atomic.h"
#include "libc/intrin/cmpxchg.h"
#include "libc/intrin/extend.internal.h"
#include "libc/intrin/strace.internal.h"
#include "libc/macros.internal.h"
#include "libc/mem/mem.h"
#include "libc/runtime/directmap.internal.h"
#include "libc/runtime/memtrack.internal.h"
#include "libc/runtime/runtime.h"
#include "libc/str/str.h"
#include "libc/sysv/consts/map.h"
#include "libc/sysv/consts/prot.h"
#include "libc/sysv/consts/sig.h"
#include "libc/sysv/errfuns.h"
// TODO(jart): make more of this code lockless

4
libc/calls/symlinkat.c
View file

@ -17,12 +17,12 @@
│ PERFORMANCE OF THIS SOFTWARE. │
╚─────────────────────────────────────────────────────────────────────────────*/
#include "libc/calls/calls.h"
#include "libc/intrin/strace.internal.h"
#include "libc/calls/syscall-nt.internal.h"
#include "libc/calls/syscall-sysv.internal.h"
#include "libc/dce.h"
#include "libc/intrin/asan.internal.h"
#include "libc/intrin/describeflags.internal.h"
#include "libc/intrin/strace.internal.h"
#include "libc/sysv/consts/at.h"
#include "libc/sysv/errfuns.h"
@ -42,7 +42,7 @@
int symlinkat(const char *target, int newdirfd, const char *linkpath) {
int rc;
if (IsAsan() &&
(!__asan_is_valid(target, 1) || !__asan_is_valid(linkpath, 1))) {
(!__asan_is_valid_str(target) || !__asan_is_valid_str(linkpath))) {
rc = efault();
}
if (!IsWindows()) {

4
libc/calls/sys_ptrace.c
View file

@ -17,11 +17,11 @@
│ PERFORMANCE OF THIS SOFTWARE. │
╚─────────────────────────────────────────────────────────────────────────────*/
#include "libc/calls/calls.h"
#include "libc/intrin/strace.internal.h"
#include "libc/calls/syscall-sysv.internal.h"
#include "libc/dce.h"
#include "libc/intrin/describeflags.internal.h"
#include "libc/intrin/likely.h"
#include "libc/intrin/strace.internal.h"
#define IsPeek(request) (IsLinux() && (request)-1u < 3)
@ -46,7 +46,7 @@ int sys_ptrace(int op, ...) {
va_end(va);
rc = __sys_ptrace(op, pid, addr, data);
#ifdef SYSDEBUG
if (UNLIKELY(__strace > 0)) {
if (UNLIKELY(__strace > 0) && strace_enabled(0) > 0) {
if (rc != -1 && IsPeek(op) && data) {
STRACE("sys_ptrace(%s, %d, %p, [%p]) → %p% m", DescribePtrace(op), pid,
addr, *data, rc);

2
libc/calls/truncate.c
View file

@ -66,7 +66,7 @@ int truncate(const char *path, int64_t length) {
struct ZiposUri zipname;
if (IsMetal()) {
rc = enosys();
} else if (!path || (IsAsan() && !__asan_is_valid(path, 1))) {
} else if (!path || (IsAsan() && !__asan_is_valid_str(path))) {
rc = efault();
} else if (_weaken(__zipos_parseuri) &&
_weaken(__zipos_parseuri)(path, &zipname) != -1) {

2
libc/calls/unlinkat.c
View file

@ -40,7 +40,7 @@
*/
int unlinkat(int dirfd, const char *path, int flags) {
int rc;
if (IsAsan() && !__asan_is_valid(path, 1)) {
if (IsAsan() && !__asan_is_valid_str(path)) {
rc = efault();
} else if (_weaken(__zipos_notat) &&
(rc = __zipos_notat(dirfd, path)) == -1) {

2
libc/calls/utimens.c
View file

@ -38,7 +38,7 @@ int __utimens(int fd, const char *path, const struct timespec ts[2],
struct ZiposUri zipname;
if (IsMetal()) {
rc = enosys();
} else if (IsAsan() && ((fd == AT_FDCWD && !__asan_is_valid(path, 1)) ||
} else if (IsAsan() && ((fd == AT_FDCWD && !__asan_is_valid_str(path)) ||
(ts && (!__asan_is_valid_timespec(ts + 0) ||
!__asan_is_valid_timespec(ts + 1))))) {
rc = efault(); // bad memory

33
libc/intrin/__pid_exec.S Normal file
View file

@ -0,0 +1,33 @@
/*-*- mode:unix-assembly; indent-tabs-mode:t; tab-width:8; coding:utf-8 -*-│
│vi: set et ft=asm ts=8 sw=8 fenc=utf-8 :vi│
╞══════════════════════════════════════════════════════════════════════════════╡
│ Copyright 2020 Justine Alexandra Roberts Tunney │
│ │
│ Permission to use, copy, modify, and/or distribute this software for │
│ any purpose with or without fee is hereby granted, provided that the │
│ above copyright notice and this permission notice appear in all copies. │
│ │
│ THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL │
│ WARRANTIES WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED │
│ WARRANTIES OF MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE │
│ AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL │
│ DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR │
│ PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER │
│ TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR │
│ PERFORMANCE OF THIS SOFTWARE. │
╚─────────────────────────────────────────────────────────────────────────────*/
#include "libc/macros.internal.h"
#include "libc/notice.inc"
// __pid_exec is __pid that isn't updated on fork()
.initbss 301,_init__pid_exec
__pid_exec:
.quad 0
.endobj __pid_exec,globl
.previous
.init.start 301,_init__pid_exec
mov __pid(%rip),%rax
stosq
.init.end 301,_init__pid_exec

339
libc/intrin/asan.c
View file

@ -16,47 +16,35 @@
│ TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR │
│ PERFORMANCE OF THIS SOFTWARE. │
╚─────────────────────────────────────────────────────────────────────────────*/
#include "libc/assert.h"
#include "libc/calls/calls.h"
#include "libc/calls/state.internal.h"
#include "libc/calls/struct/iovec.h"
#include "libc/dce.h"
#include "libc/intrin/asan.internal.h"
#include "libc/intrin/asancodes.h"
#include "libc/intrin/atomic.h"
#include "libc/intrin/cmpxchg.h"
#include "libc/intrin/directmap.internal.h"
#include "libc/intrin/kmalloc.h"
#include "libc/intrin/kprintf.h"
#include "libc/intrin/leaky.internal.h"
#include "libc/intrin/likely.h"
#include "libc/intrin/lockcmpxchg.h"
#include "libc/intrin/nomultics.internal.h"
#include "libc/intrin/strace.internal.h"
#include "libc/intrin/tpenc.h"
#include "libc/intrin/weaken.h"
#include "libc/log/backtrace.internal.h"
#include "libc/log/internal.h"
#include "libc/log/libfatal.internal.h"
#include "libc/log/log.h"
#include "libc/macros.internal.h"
#include "libc/mem/hook/hook.internal.h"
#include "libc/mem/mem.h"
#include "libc/mem/reverse.internal.h"
#include "libc/nexgen32e/gc.internal.h"
#include "libc/nexgen32e/stackframe.h"
#include "libc/nt/enum/version.h"
#include "libc/nt/runtime.h"
#include "libc/runtime/directmap.internal.h"
#include "libc/runtime/internal.h"
#include "libc/runtime/memtrack.internal.h"
#include "libc/runtime/runtime.h"
#include "libc/runtime/stack.h"
#include "libc/runtime/symbols.internal.h"
#include "libc/str/str.h"
#include "libc/str/tab.internal.h"
#include "libc/sysv/consts/auxv.h"
#include "libc/sysv/consts/map.h"
#include "libc/sysv/consts/nr.h"
#include "libc/sysv/consts/prot.h"
#include "libc/sysv/errfuns.h"
#include "libc/thread/thread.h"
#include "libc/thread/tls.h"
#include "third_party/dlmalloc/dlmalloc.h"
@ -109,6 +97,8 @@ STATIC_YOINK("_init_asan");
* movq (%addr),%dst
*/
#define RBP __builtin_frame_address(0)
#define HOOK(HOOK, IMPL) \
do { \
if (_weaken(HOOK)) { \
@ -116,17 +106,13 @@ STATIC_YOINK("_init_asan");
} \
} while (0)
#define REQUIRE(FUNC) \
do { \
if (!_weaken(FUNC)) { \
kprintf("error: asan needs %s\n", #FUNC); \
__asan_die()(); \
__asan_unreachable(); \
} \
#define REQUIRE(FUNC) \
do { \
if (!_weaken(FUNC)) { \
__asan_require_failed(#FUNC); \
} \
} while (0)
typedef char xmm_t __attribute__((__vector_size__(16), __aligned__(1)));
struct AsanTrace {
uint32_t p[ASAN_TRACE_ITEMS]; // assumes linkage into 32-bit space
};
@ -161,20 +147,22 @@ struct AsanGlobal {
char *odr_indicator;
};
struct AsanMorgue {
unsigned i;
void *p[ASAN_MORGUE_ITEMS];
};
struct ReportOriginHeap {
const unsigned char *a;
int z;
};
static char __asan_bigbuf[FRAMESIZE];
static int __asan_noreentry;
static pthread_spinlock_t __asan_lock;
static struct AsanMorgue __asan_morgue;
static struct AsanMorgue {
_Atomic(unsigned) i;
_Atomic(void *) p[ASAN_MORGUE_ITEMS];
} __asan_morgue;
static bool __asan_once(void) {
bool want = false;
static _Atomic(bool) once;
return atomic_compare_exchange_strong_explicit(
&once, &want, true, memory_order_relaxed, memory_order_relaxed);
}
#define __asan_unreachable() \
do { \
@ -363,6 +351,12 @@ dontdiscard static __asan_die_f *__asan_die(void) {
}
}
static wontreturn void __asan_require_failed(const char *func) {
kprintf("error: asan needs %s\n", func);
__asan_die()();
__asan_unreachable();
}
void __asan_poison(void *p, long n, signed char t) {
signed char k, *s;
s = (signed char *)(((intptr_t)p >> 3) + 0x7fff8000);
@ -433,17 +427,14 @@ static struct AsanFault __asan_fault(const signed char *s, signed char dflt) {
}
static struct AsanFault __asan_checka(const signed char *s, long ndiv8) {
intptr_t a;
uint64_t w;
const signed char *e = s + ndiv8;
for (; ((intptr_t)s & 7) && s < e; ++s) {
if (*s) return __asan_fault(s - 1, kAsanHeapOverrun);
}
for (; s + 8 <= e; s += 8) {
if (UNLIKELY(!((a = (intptr_t)s) & 0xffff))) {
if (kisdangerous((void *)a)) {
return (struct AsanFault){kAsanUnmapped, s};
}
if (UNLIKELY(!((intptr_t)s & (FRAMESIZE - 1))) && kisdangerous(s)) {
return (struct AsanFault){kAsanUnmapped, s};
}
if ((w = ((uint64_t)(255 & s[0]) << 000 | (uint64_t)(255 & s[1]) << 010 |
(uint64_t)(255 & s[2]) << 020 | (uint64_t)(255 & s[3]) << 030 |
@ -462,25 +453,27 @@ static struct AsanFault __asan_checka(const signed char *s, long ndiv8) {
/**
* Checks validity of memory range.
*
* Normally this is abstracted by the compiler.
* This is normally abstracted by the compiler. In some cases, it may be
* desirable to perform an ASAN memory safety check explicitly, e.g. for
* system call wrappers that need to vet memory passed to the kernel, or
* string library routines that use the `noasan` keyword due to compiler
* generated ASAN being too costly. This function is fast especially for
* large memory ranges since this takes a few picoseconds for each byte.
*
* @param p is starting virtual address
* @param n is number of bytes to check
* @return kind is 0 on success or <0 on invalid
* @return kind is 0 on success or <0 if invalid
* @return shadow points to first poisoned shadow byte
* @note this takes 6 picoseconds per byte
*/
struct AsanFault __asan_check(const void *p, long n) {
intptr_t a;
struct AsanFault f;
signed char c, k, *s;
if (n > 0) {
k = (intptr_t)p & 7;
a = ((intptr_t)p >> 3) + 0x7fff8000;
s = (signed char *)a;
s = SHADOW(p);
if (OverlapsShadowSpace(p, n)) {
return (struct AsanFault){kAsanProtected, s};
} else if (kisdangerous((void *)a)) {
} else if (kisdangerous(s)) {
return (struct AsanFault){kAsanUnmapped, s};
}
if (UNLIKELY(k)) {
@ -509,12 +502,95 @@ struct AsanFault __asan_check(const void *p, long n) {
}
}
/**
* Checks validity of nul-terminated string.
*
* This is similar to `__asan_check(p, 1)` except it checks the validity
* of the entire string including its trailing nul byte, and goes faster
* than calling strlen() beforehand.
*
* @param p is starting virtual address
* @param n is number of bytes to check
* @return kind is 0 on success or <0 if invalid
* @return shadow points to first poisoned shadow byte
*/
struct AsanFault __asan_check_str(const char *p) {
uint64_t w;
struct AsanFault f;
signed char c, k, *s;
s = SHADOW(p);
if (OverlapsShadowSpace(p, 1)) {
return (struct AsanFault){kAsanProtected, s};
}
if (kisdangerous(s)) {
return (struct AsanFault){kAsanUnmapped, s};
}
if ((k = (intptr_t)p & 7)) {
do {
if ((c = *s) && c < k + 1) {
return __asan_fault(s, kAsanHeapOverrun);
}
if (!*p++) {
return (struct AsanFault){0};
}
} while ((k = (intptr_t)p & 7));
++s;
}
for (;; ++s, p += 8) {
if (UNLIKELY(!((intptr_t)s & (FRAMESIZE - 1))) && kisdangerous(s)) {
return (struct AsanFault){kAsanUnmapped, s};
}
if ((c = *s) < 0) {
return (struct AsanFault){c, s};
}
w = *(const uint64_t *)p;
if (!(w = ~w & (w - 0x0101010101010101) & 0x8080808080808080)) {
if (c > 0) {
return __asan_fault(s, kAsanHeapOverrun);
}
} else {
k = (unsigned)__builtin_ctzll(w) >> 3;
if (!c || c > k) {
return (struct AsanFault){0};
} else {
return __asan_fault(s, kAsanHeapOverrun);
}
}
}
}
/**
* Checks memory validity of memory region.
*/
bool __asan_is_valid(const void *p, long n) {
struct AsanFault f;
f = __asan_check(p, n);
return !f.kind;
}
/**
* Checks memory validity of nul-terminated string.
*/
bool __asan_is_valid_str(const char *p) {
struct AsanFault f;
f = __asan_check_str(p);
return !f.kind;
}
/**
* Checks memory validity of null-terminated nul-terminated string list.
*/
bool __asan_is_valid_strlist(char *const *p) {
for (;; ++p) {
if (!__asan_is_valid(p, sizeof(char *))) return false;
if (!*p) return true;
if (!__asan_is_valid_str(*p)) return false;
}
}
/**
* Checks memory validity of `iov` array and each buffer it describes.
*/
bool __asan_is_valid_iov(const struct iovec *iov, int iovlen) {
int i;
size_t size;
@ -532,15 +608,7 @@ bool __asan_is_valid_iov(const struct iovec *iov, int iovlen) {
}
}
bool __asan_is_valid_strlist(char *const *p) {
for (;; ++p) {
if (!__asan_is_valid(p, sizeof(char *))) return false;
if (!*p) return true;
if (!__asan_is_valid(*p, 1)) return false;
}
}
wint_t __asan_symbolize_access_poison(signed char kind) {
static wint_t __asan_symbolize_access_poison(signed char kind) {
switch (kind) {
case kAsanNullPage:
return L'∅';
@ -589,7 +657,7 @@ wint_t __asan_symbolize_access_poison(signed char kind) {
}
}
const char *__asan_describe_access_poison(signed char kind) {
static const char *__asan_describe_access_poison(signed char kind) {
switch (kind) {
case kAsanNullPage:
return "null pointer dereference";
@ -638,7 +706,7 @@ const char *__asan_describe_access_poison(signed char kind) {
}
}
dontdiscard static __asan_die_f *__asan_report_invalid_pointer(
static dontdiscard __asan_die_f *__asan_report_invalid_pointer(
const void *addr) {
kprintf("\n\e[J\e[1;31masan error\e[0m: this corruption at %p shadow %p\n",
addr, SHADOW(addr));
@ -702,7 +770,7 @@ static void __asan_report_memory_origin_image(intptr_t a, int z) {
}
}
static noasan void OnMemory(void *x, void *y, size_t n, void *a) {
static noasan void __asan_onmemory(void *x, void *y, size_t n, void *a) {
const unsigned char *p = x;
struct ReportOriginHeap *t = a;
if ((p <= t->a && t->a < p + n) ||
@ -720,7 +788,7 @@ static void __asan_report_memory_origin_heap(const unsigned char *a, int z) {
if (_weaken(malloc_inspect_all)) {
t.a = a;
t.z = z;
_weaken(malloc_inspect_all)(OnMemory, &t);
_weaken(malloc_inspect_all)(__asan_onmemory, &t);
} else {
kprintf("\tunknown please STATIC_YOINK(\"malloc_inspect_all\");\n");
}
@ -763,10 +831,10 @@ dontdiscard static __asan_die_f *__asan_report(const void *addr, int size,
wint_t c;
signed char t;
uint64_t x, y, z;
char *p, *q, *base;
char *p, *q, *buf, *base;
struct MemoryIntervals *m;
--__ftrace;
p = __asan_bigbuf;
ftrace_enabled(-1);
p = buf = kmalloc(1024 * 1024);
kprintf("\n\e[J\e[1;31masan error\e[0m: %s %d-byte %s at %p shadow %p\n",
__asan_describe_access_poison(kind), size, message, addr,
SHADOW(addr));
@ -840,17 +908,16 @@ dontdiscard static __asan_die_f *__asan_report(const void *addr, int size,
}
__mmi_unlock();
*p = 0;
kprintf("%s", __asan_bigbuf);
kprintf("%s", buf);
__asan_report_memory_origin(addr, size, kind);
kprintf("\nthe crash was caused by\n");
++__ftrace;
ftrace_enabled(+1);
return __asan_die();
}
void __asan_verify(const void *p, size_t n) {
static wontreturn void __asan_verify_failed(const void *p, size_t n,
struct AsanFault f) {
const char *q;
struct AsanFault f;
if (!(f = __asan_check(p, n)).kind) return;
q = UNSHADOW(f.shadow);
if ((uintptr_t)q != ((uintptr_t)p & -8) && (uintptr_t)q - (uintptr_t)p < n) {
n -= (uintptr_t)q - (uintptr_t)p;
@ -860,34 +927,40 @@ void __asan_verify(const void *p, size_t n) {
__asan_unreachable();
}
dontdiscard __asan_die_f *__asan_report_memory_fault(void *addr, int size,
const char *message) {
void __asan_verify(const void *p, size_t n) {
struct AsanFault f;
if (!(f = __asan_check(p, n)).kind) return;
__asan_verify_failed(p, n, f);
}
void __asan_verify_str(const char *p) {
struct AsanFault f;
if (!(f = __asan_check_str(p)).kind) return;
__asan_verify_failed(UNSHADOW(f.shadow), 8, f);
}
static dontdiscard __asan_die_f *__asan_report_memory_fault(
void *addr, int size, const char *message) {
return __asan_report(addr, size, message,
__asan_fault(SHADOW(addr), -128).kind);
}
void *__asan_morgue_add(void *p) {
int i;
void *r;
if (__threaded) pthread_spin_lock(&__asan_lock);
i = __asan_morgue.i++ & (ARRAYLEN(__asan_morgue.p) - 1);
r = __asan_morgue.p[i];
__asan_morgue.p[i] = p;
if (__threaded) pthread_spin_unlock(&__asan_lock);
return r;
static void *__asan_morgue_add(void *p) {
return atomic_exchange_explicit(
__asan_morgue.p + (atomic_fetch_add_explicit(&__asan_morgue.i, 1,
memory_order_acquire) &
(ARRAYLEN(__asan_morgue.p) - 1)),
p, memory_order_release);
}
static void __asan_morgue_flush(void) {
int i;
void *p;
if (__threaded) pthread_spin_lock(&__asan_lock);
unsigned i;
for (i = 0; i < ARRAYLEN(__asan_morgue.p); ++i) {
if (__asan_morgue.p[i] && _weaken(dlfree)) {
_weaken(dlfree)(__asan_morgue.p[i]);
if (atomic_load_explicit(__asan_morgue.p + i, memory_order_acquire)) {
_weaken(dlfree)(atomic_exchange_explicit(__asan_morgue.p + i, 0,
memory_order_release));
}
__asan_morgue.p[i] = 0;
}
if (__threaded) pthread_spin_unlock(&__asan_lock);
}
static size_t __asan_user_size(size_t n) {
@ -963,8 +1036,8 @@ static void __asan_trace(struct AsanTrace *bt, const struct StackFrame *bp) {
#define __asan_trace __asan_rawtrace
void *__asan_allocate(size_t a, size_t n, int underrun, int overrun,
struct AsanTrace *bt) {
static void *__asan_allocate(size_t a, size_t n, struct AsanTrace *bt,
int underrun, int overrun, int initializer) {
char *p;
size_t c;
struct AsanExtra *e;
@ -975,13 +1048,17 @@ void *__asan_allocate(size_t a, size_t n, int underrun, int overrun,
__asan_unpoison(p, n);
__asan_poison(p - 16, 16, underrun); /* see dlmalloc design */
__asan_poison(p + n, c - n, overrun);
__asan_memset(p, 0xF9, n);
__asan_memset(p, initializer, n);
__asan_write48(&e->size, n);
__asan_memcpy(&e->bt, bt, sizeof(*bt));
}
return p;
}
static void *__asan_allocate_heap(size_t a, size_t n, struct AsanTrace *bt) {
return __asan_allocate(a, n, bt, kAsanHeapUnderrun, kAsanHeapOverrun, 0xf9);
}
static struct AsanExtra *__asan_get_extra(const void *p, size_t *c) {
int f;
long x, n;
@ -1003,31 +1080,20 @@ static struct AsanExtra *__asan_get_extra(const void *p, size_t *c) {
size_t __asan_get_heap_size(const void *p) {
size_t n, c;
struct AsanExtra *e;
if ((e = __asan_get_extra(p, &c))) {
if (__asan_read48(e->size, &n)) {
return n;
} else {
return 0;
}
} else {
return 0;
if ((e = __asan_get_extra(p, &c)) && __asan_read48(e->size, &n)) {
return n;
}
return 0;
}
static size_t __asan_malloc_usable_size(const void *p) {
size_t n, c;
struct AsanExtra *e;
if ((e = __asan_get_extra(p, &c))) {
if (__asan_read48(e->size, &n)) {
return n;
} else {
__asan_report_invalid_pointer(p)();
__asan_unreachable();
}
} else {
__asan_report_invalid_pointer(p)();
__asan_unreachable();
if ((e = __asan_get_extra(p, &c)) && __asan_read48(e->size, &n)) {
return n;
}
__asan_report_invalid_pointer(p)();
__asan_unreachable();
}
int __asan_print_trace(void *p) {
@ -1122,7 +1188,7 @@ static void *__asan_realloc_nogrow(void *p, size_t n, size_t m,
static void *__asan_realloc_grow(void *p, size_t n, size_t m,
struct AsanTrace *bt) {
char *q;
if ((q = __asan_allocate(16, n, kAsanHeapUnderrun, kAsanHeapOverrun, bt))) {
if ((q = __asan_allocate_heap(16, n, bt))) {
__asan_memcpy(q, p, m);
__asan_deallocate(p, kAsanHeapRelocated);
}
@ -1136,48 +1202,41 @@ static void *__asan_realloc_impl(void *p, size_t n,
struct AsanExtra *e;
if ((e = __asan_get_extra(p, &c))) {
if (__asan_read48(e->size, &m)) {
if (n <= m) { /* shrink */
if (n <= m) { // shrink
__asan_poison((char *)p + n, m - n, kAsanHeapOverrun);
__asan_write48(&e->size, n);
return p;
} else if (n <= c - sizeof(struct AsanExtra)) { /* small growth */
} else if (n <= c - sizeof(struct AsanExtra)) { // small growth
__asan_unpoison((char *)p + m, n - m);
__asan_write48(&e->size, n);
return p;
} else { /* exponential growth */
} else { // exponential growth
return grow(p, n, m, &e->bt);
}
} else {
__asan_report_invalid_pointer(p)();
__asan_unreachable();
}
} else {
__asan_report_invalid_pointer(p)();
__asan_unreachable();
}
__asan_report_invalid_pointer(p)();
__asan_unreachable();
}
void *__asan_malloc(size_t size) {
struct AsanTrace bt;
__asan_trace(&bt, __builtin_frame_address(0));
return __asan_allocate(16, size, kAsanHeapUnderrun, kAsanHeapOverrun, &bt);
__asan_trace(&bt, RBP);
return __asan_allocate_heap(16, size, &bt);
}
void *__asan_memalign(size_t align, size_t size) {
struct AsanTrace bt;
__asan_trace(&bt, __builtin_frame_address(0));
return __asan_allocate(align, size, kAsanHeapUnderrun, kAsanHeapOverrun, &bt);
__asan_trace(&bt, RBP);
return __asan_allocate_heap(align, size, &bt);
}
void *__asan_calloc(size_t n, size_t m) {
char *p;
struct AsanTrace bt;
__asan_trace(&bt, __builtin_frame_address(0));
__asan_trace(&bt, RBP);
if (__builtin_mul_overflow(n, m, &n)) n = -1;
if ((p = __asan_allocate(16, n, kAsanHeapUnderrun, kAsanHeapOverrun, &bt))) {
__asan_memset(p, 0, n);
}
return p;
return __asan_allocate(16, n, &bt, kAsanHeapUnderrun, kAsanHeapOverrun, 0x00);
}
void *__asan_realloc(void *p, size_t n) {
@ -1190,32 +1249,25 @@ void *__asan_realloc(void *p, size_t n) {
return 0;
}
} else {
__asan_trace(&bt, __builtin_frame_address(0));
return __asan_allocate(16, n, kAsanHeapUnderrun, kAsanHeapOverrun, &bt);
__asan_trace(&bt, RBP);
return __asan_allocate_heap(16, n, &bt);
}
}
void *__asan_realloc_in_place(void *p, size_t n) {
if (p) {
return __asan_realloc_impl(p, n, __asan_realloc_nogrow);
} else {
return 0;
}
return p ? __asan_realloc_impl(p, n, __asan_realloc_nogrow) : 0;
}
int __asan_malloc_trim(size_t pad) {
__asan_morgue_flush();
if (_weaken(dlmalloc_trim)) {
return _weaken(dlmalloc_trim)(pad);
} else {
return 0;
}
return _weaken(dlmalloc_trim) ? _weaken(dlmalloc_trim)(pad) : 0;
}
void *__asan_stack_malloc(size_t size, int classid) {
struct AsanTrace bt;
__asan_trace(&bt, __builtin_frame_address(0));
return __asan_allocate(16, size, kAsanStackUnderrun, kAsanStackOverrun, &bt);
__asan_trace(&bt, RBP);
return __asan_allocate(16, size, &bt, kAsanStackUnderrun, kAsanStackOverrun,
0xf9);
}
void __asan_stack_free(char *p, size_t size, int classid) {
@ -1247,14 +1299,14 @@ void __asan_unregister_globals(struct AsanGlobal g[], int n) {
void __asan_evil(uint8_t *addr, int size, const char *s) {
struct AsanTrace tr;
__asan_rawtrace(&tr, __builtin_frame_address(0));
__asan_rawtrace(&tr, RBP);
kprintf("WARNING: ASAN bad %d byte %s at %x bt %x %x %x\n", size, s, addr,
tr.p[0], tr.p[1], tr.p[2], tr.p[3]);
}
void __asan_report_load(uint8_t *addr, int size) {
__asan_evil(addr, size, "load");
if (!__vforked && _lockcmpxchg(&__asan_noreentry, 0, 1)) {
if (!__vforked && __asan_once()) {
__asan_report_memory_fault(addr, size, "load")();
__asan_unreachable();
}
@ -1262,7 +1314,7 @@ void __asan_report_load(uint8_t *addr, int size) {
void __asan_report_store(uint8_t *addr, int size) {
__asan_evil(addr, size, "store");
if (!__vforked && _lockcmpxchg(&__asan_noreentry, 0, 1)) {
if (!__vforked && __asan_once()) {
__asan_report_memory_fault(addr, size, "store")();
__asan_unreachable();
}
@ -1412,7 +1464,7 @@ static textstartup void __asan_shadow_existing_mappings(void) {
textstartup void __asan_init(int argc, char **argv, char **envp,
intptr_t *auxv) {
static bool once;
if (!_lockcmpxchg(&once, false, true)) return;
if (!_cmpxchg(&once, false, true)) return;
if (IsWindows() && NtGetVersion() < kNtVersionWindows10) {
__write_str("error: asan binaries require windows10\r\n");
_Exitr(0); /* So `make MODE=dbg test` passes w/ Windows7 */
@ -1423,6 +1475,7 @@ textstartup void __asan_init(int argc, char **argv, char **envp,
if (_weaken(hook_malloc) || _weaken(hook_calloc) || _weaken(hook_realloc) ||
_weaken(hook_realloc_in_place) || _weaken(hook_free) ||
_weaken(hook_malloc_usable_size)) {
REQUIRE(dlfree);
REQUIRE(dlmemalign);
REQUIRE(dlmalloc_usable_size);
}

7
libc/intrin/asan.internal.h
View file

@ -19,13 +19,14 @@ struct AsanFault {
void __asan_unpoison(void *, long);
void __asan_poison(void *, long, signed char);
void __asan_verify(const void *, size_t);
void __asan_verify_str(const char *);
void __asan_map_shadow(uintptr_t, size_t);
bool __asan_is_valid(const void *, long) nosideeffect;
bool __asan_is_valid_strlist(char *const *) strlenesque;
bool __asan_is_valid_str(const char *) nosideeffect;
bool __asan_is_valid_strlist(char *const *) nosideeffect;
bool __asan_is_valid_iov(const struct iovec *, int) nosideeffect;
wint_t __asan_symbolize_access_poison(signed char) pureconst;
const char *__asan_describe_access_poison(signed char) pureconst;
struct AsanFault __asan_check(const void *, long) nosideeffect;
struct AsanFault __asan_check_str(const char *) nosideeffect;
void __asan_free(void *);
void *__asan_malloc(size_t);

18
libc/intrin/assertfail.greg.c
View file

@ -17,31 +17,29 @@
│ PERFORMANCE OF THIS SOFTWARE. │
╚─────────────────────────────────────────────────────────────────────────────*/
#include "libc/assert.h"
#include "libc/calls/calls.h"
#include "libc/atomic.h"
#include "libc/calls/state.internal.h"
#include "libc/calls/syscall-sysv.internal.h"
#include "libc/intrin/atomic.h"
#include "libc/intrin/kprintf.h"
#include "libc/intrin/lockcmpxchgp.h"
#include "libc/intrin/weaken.h"
#include "libc/log/backtrace.internal.h"
#include "libc/log/internal.h"
#include "libc/runtime/internal.h"
#include "libc/runtime/runtime.h"
#include "libc/runtime/symbols.internal.h"
/**
* Handles failure of assert() macro.
*/
relegated void __assert_fail(const char *expr, const char *file, int line) {
int me, owner;
static int sync;
static atomic_int once;
if (!__assert_disable) {
--__strace;
--__ftrace;
strace_enabled(-1);
ftrace_enabled(-1);
owner = 0;
me = sys_gettid();
kprintf("%s:%d: assert(%s) failed (tid %d)\n", file, line, expr, me);
if (__vforked || _lockcmpxchgp(&sync, &owner, me)) {
if (__vforked ||
atomic_compare_exchange_strong_explicit(
&once, &owner, me, memory_order_relaxed, memory_order_relaxed)) {
__restore_tty();
if (_weaken(ShowBacktrace)) {
_weaken(ShowBacktrace)(2, __builtin_frame_address(0));

92
libc/intrin/atomic.h
View file

@ -6,11 +6,9 @@
/**
* @fileoverview Cosmopolitan C11 Atomics Library
*
* - Forty-two different ways to say MOV.
* - Fourteen different ways to say XCHG.
* - Twenty different ways to say LOCK CMPXCHG.
*
* It's a lower level programming language than assembly!
* - Forty-two different ways to say MOV.
* - Fourteen different ways to say XCHG.
* - Twenty different ways to say LOCK CMPXCHG.
*
* @see libc/atomic.h
*/
@ -23,7 +21,7 @@
#define memory_order_acq_rel 4
#define memory_order_seq_cst 5
#define ATOMIC_VAR_INIT(value) (value)
#define ATOMIC_VAR_INIT(...) __VA_ARGS__
#define atomic_is_lock_free(obj) ((void)(obj), sizeof(obj) <= sizeof(void *))
#define atomic_flag atomic_bool
@ -113,27 +111,26 @@
#define atomic_store_explicit(pObject, desired, order) \
__atomic_store_n(pObject, desired, order)
#else
#elif (__GNUC__ + 0) * 100 + (__GNUC_MINOR__ + 0) >= 401
#define atomic_init(obj, value) ((void)(*(obj) = (value)))
#define atomic_thread_fence(order) __sync_synchronize()
#define atomic_signal_fence(order) __asm__ volatile("" ::: "memory")
#define __atomic_apply_stride(object, operand) \
(((__typeof__(__atomic_val(object)))0) + (operand))
#define atomic_compare_exchange_strong_explicit(object, expected, desired, \
success, failure) \
__extension__({ \
__typeof__(expected) __ep = (expected); \
__typeof__(*__ep) __e = *__ep; \
(void)(success); \
(void)(failure); \
(_Bool)((*__ep = __sync_val_compare_and_swap(object, __e, desired)) == \
__e); \
(((__typeof__(*(object)))0) + (operand))
#define atomic_compare_exchange_strong_explicit(object, expected, desired, \
success_order, failure_order) \
__extension__({ \
__typeof__(expected) __ep = (expected); \
__typeof__(*__ep) __e = *__ep; \
(void)(success_order); \
(void)(failure_order); \
(*__ep = __sync_val_compare_and_swap(object, __e, desired)) == __e; \
})
#define atomic_compare_exchange_weak_explicit(object, expected, desired, \
success, failure) \
atomic_compare_exchange_strong_explicit(object, expected, desired, success, \
failure)
#define atomic_compare_exchange_weak_explicit(object, expected, desired, \
success_order, failure_order) \
atomic_compare_exchange_strong_explicit(object, expected, desired, \
success_order, failure_order)
#if __has_builtin(__sync_swap)
#define atomic_exchange_explicit(object, desired, order) \
((void)(order), __sync_swap(object, desired))
@ -144,7 +141,7 @@
__typeof__(desired) __d = (desired); \
(void)(order); \
__sync_synchronize(); \
__sync_lock_test_and_set(&__atomic_val(__o), __d); \
__sync_lock_test_and_set(__o, __d); \
})
#endif
#define atomic_fetch_add_explicit(object, operand, order) \
@ -164,6 +161,57 @@
#define atomic_store_explicit(object, desired, order) \
((void)atomic_exchange_explicit(object, desired, order))
#elif defined(__GNUC__) && defined(__x86__) /* x86 with gcc 4.0 and earlier */
#define atomic_init(obj, value) ((void)(*(obj) = (value)))
#define atomic_thread_fence(order) __asm__ volatile("mfence" ::: "memory")
#define atomic_signal_fence(order) __asm__ volatile("" ::: "memory")
#define atomic_compare_exchange_strong_explicit(object, expected, desired, \
success_order, failure_order) \
__extension__({ \
char DidIt; \
__typeof__(object) IfThing = (object); \
__typeof__(IfThing) IsEqualToMe = (expected); \
__typeof__(*IfThing) ReplaceItWithMe = (desired), ax; \
(void)(success_order); \
(void)(failure_order); \
__asm__ volatile("lock cmpxchg\t%3,(%1)\n\t" \
"setz\t%b0" \
: "=q"(DidIt), "=r"(IfThing), "+a"(ax) \
: "r"(ReplaceItWithMe), "2"(*IsEqualToMe) \
: "memory", "cc"); \
*IsEqualToMe = ax; \
DidIt; \
})
#define atomic_compare_exchange_weak_explicit(object, expected, desired, \
success_order, failure_order) \
atomic_compare_exchange_strong_explicit(object, expected, desired, \
success_order, failure_order)
#define atomic_exchange_explicit(object, desired, order) \
__extension__({ \
__typeof__(object) __o = (object); \
__typeof__(*__o) __d = (desired); \
(void)(order); \
__asm__ volatile("xchg\t%0,%1" : "=r"(__d), "+m"(*__o) : "0"(__d)); \
__d; \
})
#define atomic_fetch_add_explicit(object, operand, order) \
__extension__({ \
__typeof__(object) __o = (object); \
__typeof__(*__o) __d = (desired); \
(void)(order); \
__asm__ volatile("lock xadd\t%0,%1" : "=r"(__d), "+m"(*__o) : "0"(__d)); \
__d; \
})
#define atomic_fetch_sub_explicit(object, operand, order) \
atomic_fetch_add_explicit(object, -(operand), order)
#define atomic_load_explicit(object, order) \
atomic_fetch_add_explicit(object, 0, order)
#define atomic_store_explicit(object, desired, order) \
((void)atomic_exchange_explicit(object, desired, order))
#else /* non-gcc or old gcc w/o x86 */
#error "atomic operations not supported with this compiler and/or architecture"
#endif
#endif /* !(__ASSEMBLER__ + __LINKER__ + 0) */

7
libc/intrin/cmpxchg.h
View file

@ -4,7 +4,7 @@
#if !(__ASSEMBLER__ + __LINKER__ + 0)
COSMOPOLITAN_C_START_
#if defined(__GNUC__) && !defined(__STRICT_ANSI__) && defined(__x86__)
#if defined(__GNUC__) && !defined(__STRICT_ANSI__) && !defined(__x86__)
#define _cmpxchg(IFTHING, ISEQUALTOME, REPLACEITWITHME) \
({ \
bool DidIt; \
@ -17,7 +17,10 @@ COSMOPOLITAN_C_START_
: "cc"); \
DidIt; \
})
#endif /* GNUC && !ANSI && x86 */
#else
#define _cmpxchg(IFTHING, ISEQUALTOME, REPLACEITWITHME) \
(*(IFTHING) == (ISEQUALTOME) ? (*(IFTHING) = (REPLACEITWITHME), 1) : 0)
#endif
COSMOPOLITAN_C_END_
#endif /* !(__ASSEMBLER__ + __LINKER__ + 0) */

2
libc/intrin/describeiovec.c
View file

@ -62,7 +62,7 @@ const char *(DescribeIovec)(char buf[N], ssize_t rc, const struct iovec *iov,
iov[i].iov_len);
}
append("%s}", iovlen > 2 ? ", ..." : "");
append("}");
return buf;
}

59
libc/intrin/describepollfds.c Normal file
View file

@ -0,0 +1,59 @@
/*-*- mode:c;indent-tabs-mode:nil;c-basic-offset:2;tab-width:8;coding:utf-8 -*-│
│vi: set net ft=c ts=2 sts=2 sw=2 fenc=utf-8 :vi│
╞══════════════════════════════════════════════════════════════════════════════╡
│ Copyright 2021 Justine Alexandra Roberts Tunney │
│ │
│ Permission to use, copy, modify, and/or distribute this software for │
│ any purpose with or without fee is hereby granted, provided that the │
│ above copyright notice and this permission notice appear in all copies. │
│ │
│ THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL │
│ WARRANTIES WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED │
│ WARRANTIES OF MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE │
│ AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL │
│ DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR │
│ PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER │
│ TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR │
│ PERFORMANCE OF THIS SOFTWARE. │
╚─────────────────────────────────────────────────────────────────────────────*/
#include "libc/dce.h"
#include "libc/intrin/asan.internal.h"
#include "libc/intrin/describeflags.internal.h"
#include "libc/intrin/kprintf.h"
#include "libc/limits.h"
#include "libc/macros.internal.h"
#include "libc/sock/struct/pollfd.h"
#include "libc/sock/struct/pollfd.internal.h"
#define N 300
#define append(...) o += ksnprintf(buf + o, N - o, __VA_ARGS__)
const char *(DescribePollFds)(char buf[N], ssize_t rc, struct pollfd *fds,
size_t nfds) {
char b64[64];
const char *d;
int i, j, o = 0;
if (!fds) return "NULL";
if ((!IsAsan() && kisdangerous(fds)) ||
(IsAsan() && !__asan_is_valid(fds, sizeof(*fds) * nfds))) {
ksnprintf(buf, N, "%p", fds);
return buf;
}
append("{");
for (i = 0; i < nfds; ++i) {
if (i) append(", ");
append("{%d, %s", fds[i].fd, (DescribePollFlags)(b64, fds[i].events));
if (rc >= 0) {
append(", [%s]", (DescribePollFlags)(b64, fds[i].revents));
}
append("}");
}
append("}");
return buf;
}

5
libc/intrin/directmap-metal.c
View file

@ -17,8 +17,8 @@
│ PERFORMANCE OF THIS SOFTWARE. │
╚─────────────────────────────────────────────────────────────────────────────*/
#include "libc/calls/calls.h"
#include "libc/intrin/directmap.internal.h"
#include "libc/macros.internal.h"
#include "libc/runtime/directmap.internal.h"
#include "libc/runtime/pc.internal.h"
#include "libc/str/str.h"
#include "libc/sysv/consts/map.h"
@ -39,8 +39,7 @@ noasan struct DirectMap sys_mmap_metal(void *paddr, size_t size, int prot,
size = ROUNDUP(size, 4096);
addr = (uint64_t)paddr;
if (!(flags & MAP_FIXED)) {
if (!addr)
addr = 4096;
if (!addr) addr = 4096;
for (i = 0; i < size; i += 4096) {
pte = __get_virtual(mm, pml4t, addr + i, false);
if (pte && (*pte & (PAGE_V | PAGE_RSRV))) {

10
libc/intrin/directmap-nt.c
View file

@ -19,26 +19,24 @@
#include "libc/assert.h"
#include "libc/calls/internal.h"
#include "libc/calls/state.internal.h"
#include "libc/intrin/directmap.internal.h"
#include "libc/nt/enum/filemapflags.h"
#include "libc/nt/enum/pageflags.h"
#include "libc/nt/memory.h"
#include "libc/nt/runtime.h"
#include "libc/nt/struct/processmemorycounters.h"
#include "libc/nt/struct/securityattributes.h"
#include "libc/runtime/directmap.internal.h"
#include "libc/sysv/consts/map.h"
#include "libc/sysv/consts/o.h"
textwindows struct DirectMap sys_mmap_nt(void *addr, size_t size, int prot,
int flags, int fd, int64_t off) {
size_t i;
bool iscow;
int iscow;
int64_t handle;
uint32_t oldprot;
struct DirectMap dm;
struct ProtectNt fl;
const struct NtSecurityAttributes *sec;
struct NtProcessMemoryCountersEx memcount;
if (fd != -1) {
handle = g_fds.p[fd].handle;
@ -85,10 +83,8 @@ textwindows struct DirectMap sys_mmap_nt(void *addr, size_t size, int prot,
size, addr))) {
if (VirtualProtect(addr, size, __prot2nt(prot, iscow), &oldprot)) {
return dm;
} else {
return dm;
UnmapViewOfFile(dm.addr);
}
UnmapViewOfFile(dm.addr);
}
CloseHandle(dm.maphandle);
}

4
libc/intrin/directmap.c
View file

@ -16,12 +16,12 @@
│ TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR │
│ PERFORMANCE OF THIS SOFTWARE. │
╚─────────────────────────────────────────────────────────────────────────────*/
#include "libc/intrin/strace.internal.h"
#include "libc/calls/syscall-sysv.internal.h"
#include "libc/errno.h"
#include "libc/intrin/describeflags.internal.h"
#include "libc/intrin/directmap.internal.h"
#include "libc/intrin/strace.internal.h"
#include "libc/nt/runtime.h"
#include "libc/runtime/directmap.internal.h"
#include "libc/runtime/memtrack.internal.h"
/**

8
libc/runtime/directmap.internal.h → libc/intrin/directmap.internal.h
View file

@ -1,5 +1,5 @@
#ifndef COSMOPOLITAN_LIBC_RUNTIME_DIRECTMAP_H_
#define COSMOPOLITAN_LIBC_RUNTIME_DIRECTMAP_H_
#ifndef COSMOPOLITAN_LIBC_INTRIN_DIRECTMAP_H_
#define COSMOPOLITAN_LIBC_INTRIN_DIRECTMAP_H_
#if !(__ASSEMBLER__ + __LINKER__ + 0)
COSMOPOLITAN_C_START_
@ -17,8 +17,8 @@ struct DirectMap sys_mmap(void *, size_t, int, int, int, int64_t);
struct DirectMap sys_mmap_nt(void *, size_t, int, int, int, int64_t);
struct DirectMap sys_mmap_metal(void *, size_t, int, int, int, int64_t);
int sys_munmap_metal(void *, size_t);
uint32_t __prot2nt(int, bool);
int __prot2nt(int, int);
COSMOPOLITAN_C_END_
#endif /* !(__ASSEMBLER__ + __LINKER__ + 0) */
#endif /* COSMOPOLITAN_LIBC_RUNTIME_DIRECTMAP_H_ */
#endif /* COSMOPOLITAN_LIBC_INTRIN_DIRECTMAP_H_ */

2
libc/intrin/extend.c
View file

@ -22,8 +22,8 @@
#include "libc/errno.h"
#include "libc/intrin/asan.internal.h"
#include "libc/intrin/asancodes.h"
#include "libc/intrin/directmap.internal.h"
#include "libc/macros.internal.h"
#include "libc/runtime/directmap.internal.h"
#include "libc/runtime/memtrack.internal.h"
#include "libc/sysv/consts/map.h"
#include "libc/sysv/consts/prot.h"

6
libc/intrin/ftrace.c
View file

@ -30,9 +30,9 @@
* short time period should say:
*
* void foo() {
* --__ftrace;
* ftrace_enabled(-1);
* bar();
* ++__ftrace;
* ftrace_enabled(+1);
* }
*
* This way you still have some flexibility to force function tracing,
@ -40,4 +40,4 @@
* though under normal circumstances, `__ftrace` should only be either
* zero or one.
*/
atomic_int __ftrace;
int __ftrace;

43
libc/intrin/ftrace_enabled.c Normal file
View file

@ -0,0 +1,43 @@
/*-*- mode:c;indent-tabs-mode:nil;c-basic-offset:2;tab-width:8;coding:utf-8 -*-│
│vi: set net ft=c ts=2 sts=2 sw=2 fenc=utf-8 :vi│
╞══════════════════════════════════════════════════════════════════════════════╡
│ Copyright 2022 Justine Alexandra Roberts Tunney │
│ │
│ Permission to use, copy, modify, and/or distribute this software for │
│ any purpose with or without fee is hereby granted, provided that the │
│ above copyright notice and this permission notice appear in all copies. │
│ │
│ THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL │
│ WARRANTIES WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED │
│ WARRANTIES OF MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE │
│ AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL │
│ DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR │
│ PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER │
│ TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR │
│ PERFORMANCE OF THIS SOFTWARE. │
╚─────────────────────────────────────────────────────────────────────────────*/
#include "libc/runtime/runtime.h"
#include "libc/thread/tls.h"
/**
* Changes function call logging state for current thread.
*
* @param delta is added to enabled state
* @return enabled state before `delta` was applied
*/
int ftrace_enabled(int delta) {
int res;
struct CosmoTib *tib;
if (__tls_enabled) {
tib = __get_tls();
res = tib->tib_ftrace;
tib->tib_ftrace += delta;
if (!__ftrace && tib->tib_ftrace > 0) {
__ftrace = 1;
}
} else {
res = __ftrace;
__ftrace += delta;
}
return res;
}

6
libc/intrin/intrin.mk
View file

@ -44,6 +44,12 @@ $(LIBC_INTRIN_A).pkg: \
$(LIBC_INTRIN_A_OBJS) \
$(foreach x,$(LIBC_INTRIN_A_DIRECTDEPS),$($(x)_A).pkg)
# we can't use asan because:
# __strace_init() calls this before asan is initialized
o/$(MODE)/libc/intrin/strace_enabled.o: private \
OVERRIDE_COPTS += \
-fno-sanitize=address
# we can't use asan because:
# asan guard pages haven't been allocated yet
o/$(MODE)/libc/intrin/directmap.o \

18
libc/intrin/kmalloc.c
View file

@ -22,7 +22,6 @@
#include "libc/intrin/asan.internal.h"
#include "libc/intrin/atomic.h"
#include "libc/intrin/extend.internal.h"
#include "libc/intrin/kprintf.h"
#include "libc/macros.internal.h"
#include "libc/runtime/memtrack.internal.h"
#include "libc/sysv/consts/map.h"
@ -37,17 +36,18 @@ static struct {
pthread_spinlock_t lock;
} g_kmalloc;
static void kmalloc_lock(void) {
if (__threaded) pthread_spin_lock(&g_kmalloc.lock);
void __kmalloc_lock(void) {
pthread_spin_lock(&g_kmalloc.lock);
}
static void kmalloc_unlock(void) {
void __kmalloc_unlock(void) {
pthread_spin_unlock(&g_kmalloc.lock);
}
__attribute__((__constructor__)) static void kmalloc_init(void) {
pthread_atfork(kmalloc_lock, kmalloc_unlock, kmalloc_unlock);
}
#ifdef _NOPL0
#define __kmalloc_lock() _NOPL0("__threadcalls", __kmalloc_lock)
#define __kmalloc_unlock() _NOPL0("__threadcalls", __kmalloc_unlock)
#endif
/**
* Allocates permanent memory.
@ -66,7 +66,7 @@ void *kmalloc(size_t size) {
char *p, *e;
size_t i, n, t;
n = ROUNDUP(size + (IsAsan() * 8), KMALLOC_ALIGN);
kmalloc_lock();
__kmalloc_lock();
t = g_kmalloc.total;
e = g_kmalloc.endptr;
i = t;
@ -80,7 +80,7 @@ void *kmalloc(size_t size) {
} else {
p = 0;
}
kmalloc_unlock();
__kmalloc_unlock();
if (p) {
_unassert(!((intptr_t)(p + i) & (KMALLOC_ALIGN - 1)));
if (IsAsan()) __asan_poison(p + i + size, n - size, kAsanHeapOverrun);

3
libc/intrin/kmalloc.h
View file

@ -5,6 +5,9 @@ COSMOPOLITAN_C_START_
void *kmalloc(size_t) mallocesque attributeallocsize((1)) returnsaligned((8));
void __kmalloc_lock(void);
void __kmalloc_unlock(void);
COSMOPOLITAN_C_END_
#endif /* !(__ASSEMBLER__ + __LINKER__ + 0) */
#endif /* COSMOPOLITAN_LIBC_INTRIN_KMALLOC_H_ */

1
libc/intrin/kprintf.greg.c
View file

@ -31,7 +31,6 @@
#include "libc/intrin/cmpxchg.h"
#include "libc/intrin/kprintf.h"
#include "libc/intrin/likely.h"
#include "libc/intrin/lockcmpxchg.h"
#include "libc/intrin/nomultics.internal.h"
#include "libc/intrin/safemacros.internal.h"
#include "libc/intrin/weaken.h"

2
libc/intrin/memtrack.greg.c
View file

@ -22,13 +22,13 @@
#include "libc/errno.h"
#include "libc/intrin/asan.internal.h"
#include "libc/intrin/bits.h"
#include "libc/intrin/directmap.internal.h"
#include "libc/intrin/likely.h"
#include "libc/intrin/strace.internal.h"
#include "libc/log/libfatal.internal.h"
#include "libc/log/log.h"
#include "libc/macros.internal.h"
#include "libc/mem/mem.h"
#include "libc/runtime/directmap.internal.h"
#include "libc/runtime/memtrack.internal.h"
#include "libc/runtime/runtime.h"
#include "libc/str/str.h"

17
libc/intrin/mman.greg.c
View file

@ -137,8 +137,7 @@ static noasan textreal void __invert_memory(struct mman *mm, uint64_t *pml4t) {
for (i = 0; i < mm->e820n; ++i) {
uint64_t ps = mm->e820[i].addr, size = mm->e820[i].size;
/* ape/ape.S has already mapped the first 2 MiB of physical memory. */
if (ps < 0x200000 && ps + size <= 0x200000)
continue;
if (ps < 0x200000 && ps + size <= 0x200000) continue;
__invert_memory_area(mm, pml4t, ps, size, PAGE_RW | PAGE_XD);
}
}
@ -148,13 +147,13 @@ static noasan textreal void __invert_memory(struct mman *mm, uint64_t *pml4t) {
* so that assembly language routines can use it. This macro can be invoked
* from inside a function whose code is known to be emitted.
*/
#define export_offsetof(type, member) \
do { \
asm volatile(".globl \"" #type "::" #member "\"\n\t" \
".set \"" #type "::" #member "\",%c0" \
: /* no outputs */ \
: "i" (offsetof(type, member))); \
} while (0)
#define export_offsetof(type, member) \
do { \
asm volatile(".globl \"" #type "::" #member "\"\n\t" \
".set \"" #type "::" #member "\",%c0" \
: /* no outputs */ \
: "i"(offsetof(type, member))); \
} while (0)
noasan textreal void __setup_mman(struct mman *mm, uint64_t *pml4t) {
export_offsetof(struct mman, pc_drive_base_table);

9
libc/intrin/mmi_lock.c
View file

@ -31,12 +31,3 @@ void(__mmi_lock)(void) {
void(__mmi_unlock)(void) {
pthread_mutex_unlock(&__mmi_lock_obj);
}
void(__mmi_funlock)(void) {
bzero(&__mmi_lock_obj, sizeof(__mmi_lock_obj));
__mmi_lock_obj._type = PTHREAD_MUTEX_RECURSIVE;
}
__attribute__((__constructor__)) static void init(void) {
pthread_atfork(__mmi_lock, __mmi_unlock, __mmi_funlock);
}

4
libc/intrin/prot2nt.greg.c
View file

@ -16,11 +16,11 @@
│ TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR │
│ PERFORMANCE OF THIS SOFTWARE. │
╚─────────────────────────────────────────────────────────────────────────────*/
#include "libc/intrin/directmap.internal.h"
#include "libc/nt/enum/pageflags.h"
#include "libc/runtime/directmap.internal.h"
#include "libc/sysv/consts/prot.h"
privileged uint32_t __prot2nt(int prot, bool iscow) {
privileged int __prot2nt(int prot, int iscow) {
switch (prot & (PROT_READ | PROT_WRITE | PROT_EXEC)) {
case PROT_READ:
return kNtPageReadonly;

1
libc/intrin/pthread_atfork.c
View file

@ -74,6 +74,7 @@
* @param parent is run by fork() after forking happens in parent process
* @param child is run by fork() after forking happens in childe process
* @return 0 on success, or errno on error
* @raise ENOMEM if we require more vespene gas
*/
int pthread_atfork(atfork_f prepare, atfork_f parent, atfork_f child) {
if (_weaken(_pthread_atfork)) {

57
libc/intrin/pthread_key_create.c
View file

@ -17,51 +17,40 @@
│ PERFORMANCE OF THIS SOFTWARE. │
╚─────────────────────────────────────────────────────────────────────────────*/
#include "libc/errno.h"
#include "libc/intrin/bsr.h"
#include "libc/intrin/atomic.h"
#include "libc/runtime/runtime.h"
#include "libc/thread/posixthread.internal.h"
#include "libc/thread/thread.h"
#include "libc/thread/tls.h"
/**
* Allocates TLS slot.
*
* If `dtor` is non-null, then it'll be called upon thread exit when the
* key's value is nonzero. The key's value is set to zero before it gets
* called. The ordering for multiple destructor calls is unspecified.
*
* @param key is set to the allocated key on success
* @param dtor specifies an optional destructor callback
* @return 0 on success, or errno on error
* @raise EAGAIN if `PTHREAD_KEYS_MAX` keys exist
*/
int pthread_key_create(pthread_key_t *key, pthread_key_dtor dtor) {
int i, j, rc = EAGAIN;
_pthread_key_lock();
for (i = 0; i < (PTHREAD_KEYS_MAX + 63) / 64; ++i) {
if (~_pthread_key_usage[i]) {
j = _bsrl(~_pthread_key_usage[i]);
_pthread_key_usage[i] |= 1ul << j;
_pthread_key_dtor[i * 64 + j] = dtor;
*key = i * 64 + j;
rc = 0;
break;
int i;
pthread_key_dtor expect;
if (!dtor) dtor = (pthread_key_dtor)-1;
for (i = 0; i < PTHREAD_KEYS_MAX; ++i) {
if (!(expect = atomic_load_explicit(_pthread_key_dtor + i,
memory_order_relaxed)) &&
atomic_compare_exchange_strong_explicit(_pthread_key_dtor + i, &expect,
dtor, memory_order_relaxed,
memory_order_relaxed)) {
*key = i;
return 0;
}
}
_pthread_key_unlock();
return rc;
}
void _pthread_key_lock(void) {
pthread_spin_lock(&_pthread_keys_lock);
}
void _pthread_key_unlock(void) {
pthread_spin_unlock(&_pthread_keys_lock);
}
static void _pthread_key_funlock(void) {
pthread_spin_init(&_pthread_keys_lock, 0);
}
static textexit void _pthread_key_atexit(void) {
_pthread_key_destruct(0);
return EAGAIN;
}
__attribute__((__constructor__)) static textstartup void _pthread_key_init() {
atexit(_pthread_key_atexit);
pthread_atfork(_pthread_key_lock, //
_pthread_key_unlock, //
_pthread_key_funlock);
atexit(_pthread_key_destruct);
}

17
libc/intrin/pthread_key_delete.c
View file

@ -17,22 +17,23 @@
│ PERFORMANCE OF THIS SOFTWARE. │
╚─────────────────────────────────────────────────────────────────────────────*/
#include "libc/errno.h"
#include "libc/intrin/atomic.h"
#include "libc/thread/posixthread.internal.h"
#include "libc/thread/thread.h"
/**
* Deletes TLS slot.
*
* @param key was created by pthread_key_create()
* @return 0 on success, or errno on error
* @raise EINVAL if `key` isn't valid
*/
int pthread_key_delete(pthread_key_t key) {
int rc;
_pthread_key_lock();
uint64_t mask;
if (key < PTHREAD_KEYS_MAX) {
_pthread_key_usage[key / 64] &= ~(1ul << (key % 64));
_pthread_key_dtor[key] = 0;
rc = 0;
atomic_store_explicit(_pthread_key_dtor + key, 0, memory_order_relaxed);
return 0;
} else {
rc = EINVAL;
return EINVAL;
}
_pthread_key_unlock();
return rc;
}

38
libc/intrin/pthread_key_destruct.c
View file

@ -16,34 +16,30 @@
│ TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR │
│ PERFORMANCE OF THIS SOFTWARE. │
╚─────────────────────────────────────────────────────────────────────────────*/
#include "libc/intrin/bsr.h"
#include "libc/intrin/atomic.h"
#include "libc/thread/posixthread.internal.h"
#include "libc/thread/thread.h"
#include "libc/thread/tls.h"
// TODO(jart): why does this even need a lock?
void _pthread_key_destruct(void *key[PTHREAD_KEYS_MAX]) {
int i, j;
uint64_t x;
void *value;
void _pthread_key_destruct(void) {
int i, j, gotsome;
void *val, **keys;
pthread_key_dtor dtor;
if (!__tls_enabled) return;
_pthread_key_lock();
if (!key) key = __get_tls()->tib_keys;
StartOver:
for (i = 0; i < (PTHREAD_KEYS_MAX + 63) / 64; ++i) {
x = _pthread_key_usage[i];
while (x) {
j = _bsrl(x);
if ((value = key[i * 64 + j]) && (dtor = _pthread_key_dtor[i * 64 + j])) {
key[i * 64 + j] = 0;
_pthread_key_unlock();
dtor(value);
_pthread_key_lock();
goto StartOver;
keys = __get_tls()->tib_keys;
for (j = 0; j < PTHREAD_DESTRUCTOR_ITERATIONS; ++j) {
for (gotsome = i = 0; i < PTHREAD_KEYS_MAX; ++i) {
if ((val = keys[i]) &&
(dtor = atomic_load_explicit(_pthread_key_dtor + i,
memory_order_relaxed)) &&
dtor != (pthread_key_dtor)-1) {
gotsome = 1;
keys[i] = 0;
dtor(val);
}
x &= ~(1ul << j);
}
if (!gotsome) {
break;
}
}
_pthread_key_unlock();
}

8
libc/intrin/pthread_keys.c
View file

@ -19,10 +19,4 @@
#include "libc/thread/posixthread.internal.h"
#include "libc/thread/thread.h"
pthread_spinlock_t _pthread_keys_lock;
// bitset of tls key registrations
uint64_t _pthread_key_usage[(PTHREAD_KEYS_MAX + 63) / 64];
// pthread tls key destructors
pthread_key_dtor _pthread_key_dtor[PTHREAD_KEYS_MAX];
_Atomic(pthread_key_dtor) _pthread_key_dtor[PTHREAD_KEYS_MAX];

24
libc/intrin/restorewintty.c
View file

@ -22,24 +22,24 @@
#include "libc/nt/runtime.h"
#include "libc/runtime/internal.h"
uint32_t __winmainpid;
__msabi extern typeof(GetCurrentProcessId) *const __imp_GetCurrentProcessId;
__msabi extern typeof(SetConsoleMode) *const __imp_SetConsoleMode;
__msabi extern typeof(GetStdHandle) *const __imp_GetStdHandle;
const char kConsoleHandles[3] = {
extern uint32_t __pid_exec;
const unsigned char kConsoleHandles[3] = {
kNtStdInputHandle,
kNtStdOutputHandle,
kNtStdErrorHandle,
};
/**
* Puts cmd.exe gui back the way it was.
*/
noinstrument void _restorewintty(void) {
int i;
// Puts cmd.exe gui back the way it was.
privileged noinstrument void _restorewintty(void) {
if (!IsWindows()) return;
if (GetCurrentProcessId() == __winmainpid) {
for (i = 0; i < 3; ++i) {
SetConsoleMode(GetStdHandle(kConsoleHandles[i]), __ntconsolemode[i]);
}
__winmainpid = 0;
if (__imp_GetCurrentProcessId() != __pid_exec) return;
for (int i = 0; i < 3; ++i) {
__imp_SetConsoleMode(__imp_GetStdHandle(kConsoleHandles[i]),
__ntconsolemode[i]);
}
}

10
libc/intrin/strace.internal.h
View file

@ -16,11 +16,11 @@
COSMOPOLITAN_C_START_
#ifdef SYSDEBUG
#define STRACE(FMT, ...) \
do { \
if (UNLIKELY(__strace > 0)) { \
__stracef(STRACE_PROLOGUE FMT "\n", ##__VA_ARGS__); \
} \
#define STRACE(FMT, ...) \
do { \
if (UNLIKELY(__strace > 0) && strace_enabled(0) > 0) { \
__stracef(STRACE_PROLOGUE FMT "\n", ##__VA_ARGS__); \
} \
} while (0)
#else
#define STRACE(FMT, ...) (void)0

43
libc/intrin/strace_enabled.c Normal file
View file

@ -0,0 +1,43 @@
/*-*- mode:c;indent-tabs-mode:nil;c-basic-offset:2;tab-width:8;coding:utf-8 -*-│
│vi: set net ft=c ts=2 sts=2 sw=2 fenc=utf-8 :vi│
╞══════════════════════════════════════════════════════════════════════════════╡
│ Copyright 2022 Justine Alexandra Roberts Tunney │
│ │
│ Permission to use, copy, modify, and/or distribute this software for │
│ any purpose with or without fee is hereby granted, provided that the │
│ above copyright notice and this permission notice appear in all copies. │
│ │
│ THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL │
│ WARRANTIES WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED │
│ WARRANTIES OF MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE │
│ AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL │
│ DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR │
│ PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER │
│ TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR │
│ PERFORMANCE OF THIS SOFTWARE. │
╚─────────────────────────────────────────────────────────────────────────────*/
#include "libc/runtime/runtime.h"
#include "libc/thread/tls.h"
/**
* Changes system call logging state for current thread.
*
* @param delta is added to enabled state
* @return enabled state before `delta` was applied
*/
int strace_enabled(int delta) {
int res;
struct CosmoTib *tib;
if (__tls_enabled) {
tib = __get_tls();
res = tib->tib_strace;
tib->tib_strace += delta;
if (!__strace && tib->tib_strace > 0) {
__strace = 1;
}
} else {
res = __strace;
__strace += delta;
}
return res;
}

5
libc/intrin/stracef.greg.c
View file

@ -19,10 +19,13 @@
#include "libc/intrin/kprintf.h"
#include "libc/intrin/strace.internal.h"
#include "libc/runtime/runtime.h"
#include "libc/thread/tls.h"
privileged void __stracef(const char *fmt, ...) {
va_list v;
if (__strace <= 0) return;
if (__strace <= 0 || (__tls_enabled && __get_tls()->tib_strace <= 0)) {
return;
}
va_start(v, fmt);
kvprintf(fmt, v);
va_end(v);

3
libc/intrin/strlen.c
View file

@ -34,10 +34,9 @@ noasan size_t strlen(const char *s) {
xmm_t z = {0};
unsigned m, k = (uintptr_t)s & 15;
const xmm_t *p = (const xmm_t *)((uintptr_t)s & -16);
if (IsAsan()) __asan_verify(s, 1);
if (IsAsan()) __asan_verify_str(s);
m = __builtin_ia32_pmovmskb128(*p == z) >> k << k;
while (!m) m = __builtin_ia32_pmovmskb128(*++p == z);
n = (const char *)p + __builtin_ctzl(m) - s;
if (IsAsan()) __asan_verify(s, n);
return n;
}

2
libc/intrin/wsarecv.c
View file

@ -52,7 +52,7 @@ textwindows int WSARecv(
if (rc == -1) {
__winerr();
}
if (UNLIKELY(__strace > 0)) {
if (UNLIKELY(__strace > 0) && strace_enabled(0) > 0) {
kprintf(STRACE_PROLOGUE "WSARecv(%lu, [", s);
DescribeIovNt(inout_lpBuffers, dwBufferCount,
rc != -1 ? NumberOfBytesRecvd : 0);

3
libc/intrin/wsarecvfrom.c
View file

@ -23,6 +23,7 @@
#include "libc/intrin/strace.internal.h"
#include "libc/nt/thunk/msabi.h"
#include "libc/nt/winsock.h"
#include "libc/runtime/runtime.h"
__msabi extern typeof(WSARecvFrom) *const __imp_WSARecvFrom;
@ -54,7 +55,7 @@ textwindows int WSARecvFrom(
if (rc == -1) {
__winerr();
}
if (UNLIKELY(__strace > 0)) {
if (UNLIKELY(__strace > 0) && strace_enabled(0) > 0) {
kprintf(STRACE_PROLOGUE "WSARecvFrom(%lu, [", s);
DescribeIovNt(inout_lpBuffers, dwBufferCount,
rc != -1 ? NumberOfBytesRecvd : 0);

29
libc/log/backtrace2.c
View file

@ -17,38 +17,25 @@
│ PERFORMANCE OF THIS SOFTWARE. │
╚─────────────────────────────────────────────────────────────────────────────*/
#include "libc/calls/calls.h"
#include "libc/calls/internal.h"
#include "libc/calls/struct/rusage.internal.h"
#include "libc/calls/syscall-sysv.internal.h"
#include "libc/calls/syscall_support-sysv.internal.h"
#include "libc/dce.h"
#include "libc/errno.h"
#include "libc/fmt/conv.h"
#include "libc/fmt/fmt.h"
#include "libc/fmt/itoa.h"
#include "libc/intrin/kprintf.h"
#include "libc/intrin/promises.internal.h"
#include "libc/intrin/safemacros.internal.h"
#include "libc/intrin/strace.internal.h"
#include "libc/intrin/weaken.h"
#include "libc/log/backtrace.internal.h"
#include "libc/log/color.internal.h"
#include "libc/log/log.h"
#include "libc/mem/alg.h"
#include "libc/mem/bisectcarleft.internal.h"
#include "libc/mem/gc.internal.h"
#include "libc/nexgen32e/gc.internal.h"
#include "libc/nexgen32e/stackframe.h"
#include "libc/runtime/runtime.h"
#include "libc/runtime/stack.h"
#include "libc/runtime/symbols.internal.h"
#include "libc/stdio/append.h"
#include "libc/stdio/stdio.h"
#include "libc/str/str.h"
#include "libc/sysv/consts/fileno.h"
#include "libc/sysv/consts/o.h"
#include "libc/sysv/consts/sig.h"
#include "libc/thread/tls.h"
#include "libc/x/x.h"
#define kBacktraceMaxFrames 128
#define kBacktraceBufSize ((kBacktraceMaxFrames - 1) * (18 + 1))
@ -58,7 +45,6 @@ static void ShowHint(const char *s) {
}
static int PrintBacktraceUsingAddr2line(int fd, const struct StackFrame *bp) {
bool ok;
ssize_t got;
intptr_t addr;
size_t i, j, gi;
@ -110,6 +96,9 @@ static int PrintBacktraceUsingAddr2line(int fd, const struct StackFrame *bp) {
garbage = __tls_enabled ? __get_tls()->tib_garbages : 0;
gi = garbage ? garbage->i : 0;
for (frame = bp; frame && i < kBacktraceMaxFrames - 1; frame = frame->next) {
if (kisdangerous(frame)) {
return -1;
}
addr = frame->addr;
if (addr == _weakaddr("__gc")) {
do {
@ -158,7 +147,7 @@ static int PrintBacktraceUsingAddr2line(int fd, const struct StackFrame *bp) {
got -= p3 - p1;
p1 += p3 - p1;
} else {
write(2, p1, got);
sys_write(2, p1, got);
break;
}
}
@ -189,12 +178,12 @@ static int PrintBacktrace(int fd, const struct StackFrame *bp) {
void ShowBacktrace(int fd, const struct StackFrame *bp) {
#ifdef __FNO_OMIT_FRAME_POINTER__
/* asan runtime depends on this function */
--__ftrace;
--__strace;
ftrace_enabled(-1);
strace_enabled(-1);
if (!bp) bp = __builtin_frame_address(0);
PrintBacktrace(fd, bp);
++__strace;
++__ftrace;
strace_enabled(+1);
ftrace_enabled(+1);
#else
(fprintf)(stderr, "ShowBacktrace() needs these flags to show C backtrace:\n"
"\t-D__FNO_OMIT_FRAME_POINTER__\n"

4
libc/log/backtrace3.c
View file

@ -58,6 +58,10 @@ noinstrument noasan int PrintBacktraceUsingSymbols(int fd,
garbage = __tls_enabled ? __get_tls()->tib_garbages : 0;
gi = garbage ? garbage->i : 0;
for (i = 0, frame = bp; frame; frame = frame->next) {
if (kisdangerous(frame)) {
kprintf("<dangerous frame>\n");
break;
}
if (++i == LIMIT) {
kprintf("<truncated backtrace>\n");
break;

4
libc/log/checkfail.c
View file

@ -45,8 +45,8 @@ relegated void __check_fail(const char *suffix, const char *opstr,
size_t i;
va_list va;
char hostname[32];
--__strace;
--__ftrace;
strace_enabled(-1);
ftrace_enabled(-1);
e = errno;
__start_fatal(file, line);
__stpcpy(hostname, "unknown");

10
libc/log/die.c
View file

@ -16,11 +16,13 @@
│ TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR │
│ PERFORMANCE OF THIS SOFTWARE. │
╚─────────────────────────────────────────────────────────────────────────────*/
#include "libc/atomic.h"
#include "libc/calls/calls.h"
#include "libc/calls/state.internal.h"
#include "libc/calls/syscall-sysv.internal.h"
#include "libc/dce.h"
#include "libc/intrin/atomic.h"
#include "libc/intrin/kprintf.h"
#include "libc/intrin/lockcmpxchgp.h"
#include "libc/log/backtrace.internal.h"
#include "libc/log/internal.h"
#include "libc/log/libfatal.internal.h"
@ -40,10 +42,12 @@ STATIC_YOINK("_idt");
relegated wontreturn void __die(void) {
/* asan runtime depends on this function */
int me, owner;
static int sync;
static atomic_int once;
owner = 0;
me = sys_gettid();
if (_lockcmpxchgp(&sync, &owner, me)) {
if (__vforked ||
atomic_compare_exchange_strong_explicit(
&once, &owner, me, memory_order_relaxed, memory_order_relaxed)) {
__restore_tty();
if (IsDebuggerPresent(false)) {
DebugBreak();

4
libc/log/leaks.c
View file

@ -19,8 +19,8 @@
#include "libc/dce.h"
#include "libc/intrin/asan.internal.h"
#include "libc/intrin/bits.h"
#include "libc/intrin/cmpxchg.h"
#include "libc/intrin/kprintf.h"
#include "libc/intrin/lockcmpxchg.h"
#include "libc/intrin/strace.internal.h"
#include "libc/mem/mem.h"
#include "libc/runtime/internal.h"
@ -82,7 +82,7 @@ static noasan bool HasLeaks(void) {
noasan void CheckForMemoryLeaks(void) {
struct mallinfo mi;
if (!IsAsan()) return; // we need traces to exclude leaky
if (!_lockcmpxchg(&once, false, true)) {
if (!_cmpxchg(&once, false, true)) {
kprintf("CheckForMemoryLeaks() may only be called once\n");
exit(1);
}

2
libc/log/log_retrace.c
View file

@ -20,5 +20,5 @@
#include "libc/runtime/runtime.h"
noinstrument void _log_retrace(void) {
++__ftrace;
ftrace_enabled(+1);
}

2
libc/log/log_untrace.c
View file

@ -20,5 +20,5 @@
#include "libc/runtime/runtime.h"
noinstrument void _log_untrace(void) {
--__ftrace;
ftrace_enabled(-1);
}

99
libc/log/oncrash.c
View file

@ -16,6 +16,8 @@
│ TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR │
│ PERFORMANCE OF THIS SOFTWARE. │
╚─────────────────────────────────────────────────────────────────────────────*/
#include "libc/assert.h"
#include "libc/atomic.h"
#include "libc/calls/calls.h"
#include "libc/calls/state.internal.h"
#include "libc/calls/struct/sigaction.h"
@ -25,9 +27,9 @@
#include "libc/errno.h"
#include "libc/fmt/itoa.h"
#include "libc/intrin/asan.internal.h"
#include "libc/intrin/atomic.h"
#include "libc/intrin/kmalloc.h"
#include "libc/intrin/kprintf.h"
#include "libc/intrin/lockcmpxchg.h"
#include "libc/intrin/lockcmpxchgp.h"
#include "libc/intrin/strace.internal.h"
#include "libc/intrin/weaken.h"
#include "libc/log/backtrace.internal.h"
@ -118,14 +120,12 @@ static char *HexCpy(char p[hasatleast 17], uint64_t x, uint8_t k) {
return p;
}
relegated static void ShowGeneralRegisters(ucontext_t *ctx) {
relegated static char *ShowGeneralRegisters(char *p, ucontext_t *ctx) {
int64_t x;
const char *s;
size_t i, j, k;
long double st;
char *p, buf[128];
p = buf;
kprintf("\n");
*p++ = '\n';
for (i = 0, j = 0, k = 0; i < ARRAYLEN(kGregNames); ++i) {
if (j > 0) *p++ = ' ';
if (!(s = kGregNames[(unsigned)kGregOrder[i]])[2]) *p++ = ' ';
@ -145,25 +145,25 @@ relegated static void ShowGeneralRegisters(ucontext_t *ctx) {
if (x < 0) x = -x, *p++ = '-';
p = FormatUint64(p, x / 1000), *p++ = '.';
p = FormatUint64(p, x % 1000);
*p = 0;
kprintf("%s\n", buf);
p = buf;
*p++ = '\n';
}
}
DescribeCpuFlags(
p, ctx->uc_mcontext.eflags,
ctx->uc_mcontext.fpregs ? ctx->uc_mcontext.fpregs->swd : 0,
ctx->uc_mcontext.fpregs ? ctx->uc_mcontext.fpregs->mxcsr : 0);
kprintf("%s\n", buf);
*p++ = '\n';
return p;
}
relegated static void ShowSseRegisters(ucontext_t *ctx) {
relegated static char *ShowSseRegisters(char *p, ucontext_t *ctx) {
size_t i;
char *p, buf[128];
if (ctx->uc_mcontext.fpregs) {
kprintf("\n");
*p++ = '\n';
for (i = 0; i < 8; ++i) {
p = buf;
*p++ = 'X';
*p++ = 'M';
*p++ = 'M';
if (i >= 10) {
*p++ = i / 10 + '0';
*p++ = i % 10 + '0';
@ -185,10 +185,10 @@ relegated static void ShowSseRegisters(ucontext_t *ctx) {
*p++ = ' ';
p = HexCpy(p, ctx->uc_mcontext.fpregs->xmm[i + 8].u64[1], 64);
p = HexCpy(p, ctx->uc_mcontext.fpregs->xmm[i + 8].u64[0], 64);
*p = 0;
kprintf("XMM%s\n", buf);
*p++ = '\n';
}
}
return p;
}
void ShowCrashReportHook(int, int, int, struct siginfo *, ucontext_t *);
@ -196,10 +196,10 @@ void ShowCrashReportHook(int, int, int, struct siginfo *, ucontext_t *);
relegated void ShowCrashReport(int err, int sig, struct siginfo *si,
ucontext_t *ctx) {
int i;
char *p;
size_t n;
char host[64];
char *p, *buf;
struct utsname names;
static char buf[4096];
if (_weaken(ShowCrashReportHook)) {
ShowCrashReportHook(2, err, sig, si, ctx);
}
@ -210,24 +210,31 @@ relegated void ShowCrashReport(int err, int sig, struct siginfo *si,
stpcpy(host, "unknown");
gethostname(host, sizeof(host));
uname(&names);
p = buf;
errno = err;
kprintf("\n%serror%s: Uncaught %G (%s) on %s pid %d tid %d\n"
" %s\n"
" %m\n"
" %s %s %s %s\n",
!__nocolor ? "\e[30;101m" : "", !__nocolor ? "\e[0m" : "", sig,
(ctx && (ctx->uc_mcontext.rsp >= GetStaticStackAddr(0) &&
ctx->uc_mcontext.rsp <= GetStaticStackAddr(0) + PAGESIZE))
? "Stack Overflow"
: GetSiCodeName(sig, si->si_code),
host, getpid(), gettid(), program_invocation_name, names.sysname,
names.version, names.nodename, names.release);
// TODO(jart): Buffer the WHOLE crash report with backtrace for atomic write.
_npassert((p = buf = kmalloc((n = 1024 * 1024))));
p += ksnprintf(
p, n,
"\n%serror%s: Uncaught %G (%s) on %s pid %d tid %d\n"
" %s\n"
" %m\n"
" %s %s %s %s\n",
!__nocolor ? "\e[30;101m" : "", !__nocolor ? "\e[0m" : "", sig,
(ctx && (ctx->uc_mcontext.rsp >= GetStaticStackAddr(0) &&
ctx->uc_mcontext.rsp <= GetStaticStackAddr(0) + PAGESIZE))
? "Stack Overflow"
: GetSiCodeName(sig, si->si_code),
host, getpid(), gettid(), program_invocation_name, names.sysname,
names.version, names.nodename, names.release);
if (ctx) {
ShowGeneralRegisters(ctx);
ShowSseRegisters(ctx);
kprintf("\n");
p = ShowGeneralRegisters(p, ctx);
p = ShowSseRegisters(p, ctx);
*p++ = '\n';
write(2, buf, p - buf);
ShowFunctionCalls(ctx);
} else {
*p++ = '\n';
write(2, buf, p - buf);
}
kprintf("\n");
if (!IsWindows()) __print_maps();
@ -235,7 +242,7 @@ relegated void ShowCrashReport(int err, int sig, struct siginfo *si,
if (__argv) {
for (i = 0; i < __argc; ++i) {
if (!__argv[i]) continue;
if (IsAsan() && !__asan_is_valid(__argv[i], 1)) continue;
if (IsAsan() && !__asan_is_valid_str(__argv[i])) continue;
kprintf("%s ", __argv[i]);
}
}
@ -279,17 +286,19 @@ static wontreturn relegated noinstrument void __minicrash(int sig,
* @vforksafe
*/
relegated void __oncrash(int sig, struct siginfo *si, ucontext_t *ctx) {
bool bZero;
intptr_t rip;
int me, owner;
int gdbpid, err;
static int sync;
static bool _notpossible;
static atomic_int once;
static atomic_bool once2;
STRACE("__oncrash rip %x", ctx->uc_mcontext.rip);
--__ftrace;
--__strace;
ftrace_enabled(-1);
strace_enabled(-1);
owner = 0;
me = sys_gettid();
if (__vforked || _lockcmpxchgp(&sync, &owner, me)) {
if (atomic_compare_exchange_strong_explicit(
&once, &owner, me, memory_order_relaxed, memory_order_relaxed)) {
if (!__vforked) {
rip = ctx ? ctx->uc_mcontext.rip : 0;
err = errno;
@ -310,9 +319,9 @@ relegated void __oncrash(int sig, struct siginfo *si, ucontext_t *ctx) {
ShowCrashReport(err, sig, si, ctx);
_Exitr(128 + sig);
}
sync = 0;
atomic_store_explicit(&once, 0, memory_order_relaxed);
} else {
sync = 0;
atomic_store_explicit(&once, 0, memory_order_relaxed);
__minicrash(sig, si, ctx, "WHILE VFORKED");
}
} else if (sig == SIGTRAP) {
@ -320,7 +329,9 @@ relegated void __oncrash(int sig, struct siginfo *si, ucontext_t *ctx) {
goto ItsATrap;
} else if (owner == me) {
// we crashed while generating a crash report
if (_lockcmpxchg(&_notpossible, false, true)) {
bZero = false;
if (atomic_compare_exchange_strong_explicit(
&once2, &bZero, true, memory_order_relaxed, memory_order_relaxed)) {
__minicrash(sig, si, ctx, "WHILE CRASHING");
} else {
// somehow __minicrash() crashed not possible
@ -335,6 +346,6 @@ relegated void __oncrash(int sig, struct siginfo *si, ucontext_t *ctx) {
_Exit1(8);
}
ItsATrap:
++__strace;
++__ftrace;
strace_enabled(+1);
ftrace_enabled(+1);
}

4
libc/log/showcrashreports.c
View file

@ -68,7 +68,7 @@ relegated void RestoreDefaultCrashSignalHandlers(void) {
int e;
size_t i;
sigset_t ss;
--__strace;
strace_enabled(-1);
sigemptyset(&ss);
sigprocmask(SIG_SETMASK, &ss, NULL);
for (i = 0; i < ARRAYLEN(kCrashSigs); ++i) {
@ -78,7 +78,7 @@ relegated void RestoreDefaultCrashSignalHandlers(void) {
errno = e;
}
}
++__strace;
strace_enabled(+1);
}
static void FreeSigAltStack(void *p) {

4
libc/log/vflogf.c
View file

@ -95,7 +95,7 @@ void(vflogf)(unsigned level, const char *file, int line, FILE *f,
if (!f) f = __log_file;
if (!f) return;
flockfile(f);
--__strace;
strace_enabled(-1);
// We display TIMESTAMP.MICROS normally. However, when we log multiple
// times in the same second, we display TIMESTAMP+DELTAMICROS instead.
@ -138,6 +138,6 @@ void(vflogf)(unsigned level, const char *file, int line, FILE *f,
unreachable;
}
++__strace;
strace_enabled(+1);
funlockfile(f);
}

8
libc/runtime/arch_prctl.c
View file

@ -17,7 +17,6 @@
│ PERFORMANCE OF THIS SOFTWARE. │
╚─────────────────────────────────────────────────────────────────────────────*/
#include "libc/calls/calls.h"
#include "libc/intrin/strace.internal.h"
#include "libc/calls/syscall-sysv.internal.h"
#include "libc/dce.h"
#include "libc/errno.h"
@ -25,6 +24,7 @@
#include "libc/intrin/asmflag.h"
#include "libc/intrin/bits.h"
#include "libc/intrin/describeflags.internal.h"
#include "libc/intrin/strace.internal.h"
#include "libc/nexgen32e/msr.h"
#include "libc/nexgen32e/x86feature.h"
#include "libc/runtime/pc.internal.h"
@ -47,7 +47,7 @@
"d"((uint32_t)(val_ >> 32))); \
} while (0)
int sys_enable_tls();
int sys_set_tls();
static int arch_prctl_msr(int code, int64_t addr) {
switch (code) {
@ -96,7 +96,7 @@ static int arch_prctl_netbsd(int code, int64_t addr) {
// we use _lwp_setprivate() instead of sysarch(X86_SET_FSBASE)
// because the latter has a bug where signal handlers cause it
// to be clobbered. please note, this doesn't apply to %gs :-)
return sys_enable_tls(addr);
return sys_set_tls(addr);
case ARCH_GET_GS:
// sysarch(X86_GET_GSBASE)
return sys_arch_prctl(14, addr);
@ -114,7 +114,7 @@ static int arch_prctl_xnu(int code, int64_t addr) {
case ARCH_SET_GS:
// thread_fast_set_cthread_self has a weird ABI
e = errno;
sys_enable_tls(addr);
sys_set_tls(addr);
errno = e;
return 0;
case ARCH_GET_FS:

2
libc/runtime/brk.c
View file

@ -20,11 +20,11 @@
#include "libc/calls/calls.h"
#include "libc/calls/syscall-sysv.internal.h"
#include "libc/dce.h"
#include "libc/intrin/directmap.internal.h"
#include "libc/intrin/nopl.internal.h"
#include "libc/intrin/strace.internal.h"
#include "libc/macros.internal.h"
#include "libc/runtime/brk.internal.h"
#include "libc/runtime/directmap.internal.h"
#include "libc/runtime/memtrack.internal.h"
#include "libc/runtime/runtime.h"
#include "libc/sysv/consts/map.h"

154
libc/runtime/enable_tls.c
View file

@ -16,52 +16,24 @@
│ TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR │
│ PERFORMANCE OF THIS SOFTWARE. │
╚─────────────────────────────────────────────────────────────────────────────*/
#include "ape/sections.internal.h"
#include "libc/assert.h"
#include "libc/calls/calls.h"
#include "libc/calls/struct/sigset.h"
#include "libc/calls/syscall-sysv.internal.h"
#include "libc/dce.h"
#include "libc/errno.h"
#include "libc/intrin/asan.internal.h"
#include "libc/intrin/asancodes.h"
#include "libc/intrin/atomic.h"
#include "libc/intrin/bits.h"
#include "libc/intrin/weaken.h"
#include "libc/log/libfatal.internal.h"
#include "libc/macros.internal.h"
#include "libc/nexgen32e/msr.h"
#include "libc/nt/thread.h"
#include "libc/runtime/internal.h"
#include "libc/runtime/morph.h"
#include "libc/runtime/runtime.h"
#include "libc/stdalign.internal.h"
#include "libc/str/str.h"
#include "libc/sysv/consts/nrlinux.h"
#include "libc/thread/posixthread.internal.h"
#include "libc/thread/tls.h"
#include "third_party/xed/x86.h"
#define __NR_sysarch 0x000000a5 // freebsd+netbsd
#define AMD64_SET_GSBASE 131 // freebsd
#define AMD64_SET_FSBASE 129 // freebsd
#define X86_SET_GSBASE 16 // netbsd
#define X86_SET_FSBASE 17 // netbsd
#define __NR___set_tcb 0x00000149
#define __NR__lwp_setprivate 0x0000013d
#define __NR_thread_fast_set_cthread_self 0x03000003
#define _TLSZ ((intptr_t)_tls_size)
#define _TLDZ ((intptr_t)_tdata_size)
#define _TIBZ sizeof(struct CosmoTib)
int sys_enable_tls();
typedef char xmm_t __attribute__((__vector_size__(16), __aligned__(1)));
__msabi extern typeof(TlsAlloc) *const __imp_TlsAlloc;
struct PosixThread _pthread_main;
extern unsigned char __tls_mov_nt_rax[];
extern unsigned char __tls_add_nt_rax[];
@ -102,12 +74,12 @@ _Alignas(TLS_ALIGNMENT) static char __static_tls[5008];
* arch_prctl() function. However, such programs might not be portable
* and your `errno` variable also won't be thread safe anymore.
*/
privileged void __enable_tls(void) {
void __enable_tls(void) {
int tid;
size_t siz;
struct CosmoTib *tib;
char *mem, *tls;
siz = ROUNDUP(_TLSZ + _TIBZ, alignof(__static_tls));
siz = ROUNDUP(_TLSZ + _TIBZ, _Alignof(__static_tls));
if (siz <= sizeof(__static_tls)) {
// if tls requirement is small then use the static tls block
// which helps avoid a system call for appes with little tls
@ -134,6 +106,8 @@ privileged void __enable_tls(void) {
tib->tib_self = tib;
tib->tib_self2 = tib;
tib->tib_errno = __errno;
tib->tib_strace = __strace;
tib->tib_ftrace = __ftrace;
tib->tib_pthread = (pthread_t)&_pthread_main;
if (IsLinux()) {
// gnu/systemd guarantees pid==tid for the main thread so we can
@ -149,124 +123,10 @@ privileged void __enable_tls(void) {
__repmovsb(tls, _tdata_start, _TLDZ);
// ask the operating system to change the x86 segment register
int ax, dx;
if (IsWindows()) {
__tls_index = __imp_TlsAlloc();
_npassert(0 <= __tls_index && __tls_index < 64);
asm("mov\t%1,%%gs:%0" : "=m"(*((long *)0x1480 + __tls_index)) : "r"(tib));
} else if (IsFreebsd()) {
sys_enable_tls(AMD64_SET_FSBASE, tib);
} else if (IsLinux()) {
sys_enable_tls(ARCH_SET_FS, tib);
} else if (IsNetbsd()) {
// netbsd has sysarch(X86_SET_FSBASE) but we can't use that because
// signal handlers will cause it to be reset due to not setting the
// _mc_tlsbase field in struct mcontext_netbsd.
sys_enable_tls(tib);
} else if (IsOpenbsd()) {
sys_enable_tls(tib);
} else if (IsXnu()) {
// thread_fast_set_cthread_self has a weird ABI
int e = errno;
sys_enable_tls((intptr_t)tib - 0x30);
errno = e;
} else {
uint64_t val = (uint64_t)tib;
asm volatile("wrmsr"
: /* no outputs */
: "c"(MSR_IA32_FS_BASE), "a"((uint32_t)val),
"d"((uint32_t)(val >> 32)));
}
__set_tls(tib);
// We need to rewrite SysV _Thread_local code. You MUST use the
// -mno-tls-direct-seg-refs flag which generates code like this
//
// 64 48 8b 0R4 25 00 00 00 00 mov %fs:0,%R
// 64 48 03 0R4 25 00 00 00 00 add %fs:0,%R
//
// Which on Mac we can replace with this:
//
// 65 48 8b 0R4 25 30 00 00 00 mov %gs:0x30,%R
//
// Since we have no idea where the TLS instructions exist in the
// binary, we need to disassemble the whole program image. This'll
// potentially take a few milliseconds for some larger programs.
//
// We check `_tls_content` which is generated by the linker script
// since it lets us determine ahead of time if _Thread_local vars
// have actually been linked into this program.
if ((intptr_t)_tls_content && (IsWindows() || IsXnu())) {
int n;
uint64_t w;
sigset_t mask;
unsigned m, dis;
unsigned char *p;
__morph_begin(&mask);
if (IsXnu()) {
// Apple is quite straightforward to patch. We basically
// just change the segment register, and the linear slot
// address 0x30 was promised to us, according to Go team
// https://github.com/golang/go/issues/23617
dis = 0x30;
} else {
// MSVC __declspec(thread) generates binary code for this
// %gs:0x1480 abi. So long as TlsAlloc() isn't called >64
// times we should be good.
dis = 0x1480 + __tls_index * 8;
}
// iterate over modifiable code looking for 9 byte instruction
// this would take 30 ms using xed to enable tls on python.com
for (p = _ereal; p + 9 <= __privileged_start; p += n) {
// use sse to zoom zoom to fs register prefixes
// that way it'll take 1 ms to morph python.com
while (p + 9 + 16 <= __privileged_start) {
if ((m = __builtin_ia32_pmovmskb128(
*(xmm_t *)p == (xmm_t){0144, 0144, 0144, 0144, 0144, 0144,
0144, 0144, 0144, 0144, 0144, 0144,
0144, 0144, 0144, 0144}))) {
m = __builtin_ctzll(m);
p += m;
break;
} else {
p += 16;
}
}
// we're checking for the following expression:
// 0144 == p[0] && // %fs
// 0110 == (p[1] & 0373) && // rex.w (and ignore rex.r)
// (0213 == p[2] || // mov reg/mem → reg (word-sized)
// 0003 == p[2]) && // add reg/mem → reg (word-sized)
// 0004 == (p[3] & 0307) && // mod/rm (4,reg,0) means sib → reg
// 0045 == p[4] && // sib (5,4,0) → (rbp,rsp,0) → disp32
// 0000 == p[5] && // displacement (von Neumann endian)
// 0000 == p[6] && // displacement
// 0000 == p[7] && // displacement
// 0000 == p[8] // displacement
w = READ64LE(p) & READ64LE("\377\373\377\307\377\377\377\377");
if ((w == READ64LE("\144\110\213\004\045\000\000\000") ||
w == READ64LE("\144\110\003\004\045\000\000\000")) &&
!p[8]) {
// now change the code
p[0] = 0145; // change %fs to %gs
p[5] = (dis & 0x000000ff) >> 000; // displacement
p[6] = (dis & 0x0000ff00) >> 010; // displacement
p[7] = (dis & 0x00ff0000) >> 020; // displacement
p[8] = (dis & 0xff000000) >> 030; // displacement
// advance to the next instruction
n = 9;
} else {
n = 1;
}
}
__morph_end(&mask);
}
// rewrite the executable tls opcodes in memory
__morph_tls();
// we are now allowed to use tls
__tls_enabled = true;

17
libc/runtime/fork-nt.c
View file

@ -24,6 +24,7 @@
#include "libc/calls/wincrash.internal.h"
#include "libc/errno.h"
#include "libc/fmt/itoa.h"
#include "libc/intrin/directmap.internal.h"
#include "libc/intrin/kprintf.h"
#include "libc/intrin/strace.internal.h"
#include "libc/intrin/weaken.h"
@ -46,7 +47,6 @@
#include "libc/nt/runtime.h"
#include "libc/nt/signals.h"
#include "libc/nt/struct/ntexceptionpointers.h"
#include "libc/runtime/directmap.internal.h"
#include "libc/runtime/internal.h"
#include "libc/runtime/memtrack.internal.h"
#include "libc/runtime/runtime.h"
@ -61,6 +61,7 @@ STATIC_YOINK("_check_sigchld");
extern int64_t __wincrashearly;
bool32 __onntconsoleevent_nt(uint32_t);
void kmalloc_unlock(void);
static textwindows wontreturn void AbortFork(const char *func) {
STRACE("fork() %s() failed %d", func, GetLastError());
@ -259,19 +260,19 @@ textwindows void WinMainForked(void) {
}
textwindows int sys_fork_nt(uint32_t dwCreationFlags) {
bool ok;
jmp_buf jb;
uint32_t oldprot;
char ok, threaded;
char **args, **args2;
struct CosmoTib *tib;
char16_t pipename[64];
bool needtls, threaded;
int64_t reader, writer;
struct NtStartupInfo startinfo;
int i, n, pid, untrackpid, rc = -1;
char *p, forkvar[6 + 21 + 1 + 21 + 1];
struct NtProcessInformation procinfo;
threaded = __threaded;
needtls = __tls_enabled;
tib = __tls_enabled ? __get_tls() : 0;
if (!setjmp(jb)) {
pid = untrackpid = __reservefd_unlocked(-1);
reader = CreateNamedPipe(CreatePipeName(pipename),
@ -293,7 +294,7 @@ textwindows int sys_fork_nt(uint32_t dwCreationFlags) {
#ifdef SYSDEBUG
// If --strace was passed to this program, then propagate it the
// forked process since the flag was removed by __intercept_flag
if (__strace > 0) {
if (strace_enabled(0) > 0) {
for (n = 0; args[n];) ++n;
args2 = alloca((n + 2) * sizeof(char *));
for (i = 0; i < n; ++i) args2[i] = args[i];
@ -345,8 +346,10 @@ textwindows int sys_fork_nt(uint32_t dwCreationFlags) {
}
} else {
rc = 0;
if (needtls) {
__enable_tls();
if (tib && _weaken(__set_tls) && _weaken(__morph_tls)) {
_weaken(__set_tls)(tib);
_weaken(__morph_tls)();
__tls_enabled = true;
}
if (threaded && !__threaded && _weaken(__enable_threads)) {
_weaken(__enable_threads)();

3
libc/runtime/ftraceinit.greg.c
View file

@ -19,6 +19,7 @@
#include "libc/runtime/internal.h"
#include "libc/runtime/runtime.h"
#include "libc/str/str.h"
#include "libc/thread/tls.h"
/**
* Enables plaintext function tracing if `--ftrace` flag is passed.
@ -35,7 +36,7 @@
textstartup int ftrace_init(void) {
if (__intercept_flag(&__argc, __argv, "--ftrace")) {
ftrace_install();
++__ftrace;
ftrace_enabled(+1);
}
return __argc;
}

7
libc/runtime/ftracer.c
View file

@ -72,10 +72,13 @@ static privileged inline int GetNestingLevel(struct CosmoFtrace *ft,
*/
privileged void ftracer(void) {
long stackuse;
struct CosmoFtrace *ft;
struct CosmoTib *tib;
struct StackFrame *sf;
struct CosmoFtrace *ft;
if (__tls_enabled) {
ft = &__get_tls_privileged()->tib_ftrace;
tib = __get_tls_privileged();
if (tib->tib_ftrace <= 0) return;
ft = &tib->tib_ftracer;
} else {
ft = &g_ftrace;
}

1
libc/runtime/internal.h
View file

@ -29,6 +29,7 @@ extern unsigned char _tls_size[];
extern unsigned char _tls_content[];
void _init(void) hidden;
void __morph_tls(void);
void __enable_tls(void);
void __enable_threads(void) hidden;
void *__cxa_finalize(void *) hidden;

2
libc/runtime/mmap.c
View file

@ -27,6 +27,7 @@
#include "libc/intrin/bits.h"
#include "libc/intrin/bsr.h"
#include "libc/intrin/describeflags.internal.h"
#include "libc/intrin/directmap.internal.h"
#include "libc/intrin/likely.h"
#include "libc/intrin/safemacros.internal.h"
#include "libc/intrin/strace.internal.h"
@ -39,7 +40,6 @@
#include "libc/nt/process.h"
#include "libc/nt/runtime.h"
#include "libc/nt/struct/processmemorycounters.h"
#include "libc/runtime/directmap.internal.h"
#include "libc/runtime/internal.h"
#include "libc/runtime/memtrack.internal.h"
#include "libc/runtime/runtime.h"

118
libc/runtime/morph_tls.c Normal file
View file

@ -0,0 +1,118 @@
/*-*- mode:c;indent-tabs-mode:nil;c-basic-offset:2;tab-width:8;coding:utf-8 -*-│
│vi: set net ft=c ts=2 sts=2 sw=2 fenc=utf-8 :vi│
╞══════════════════════════════════════════════════════════════════════════════╡
│ Copyright 2022 Justine Alexandra Roberts Tunney │
│ │
│ Permission to use, copy, modify, and/or distribute this software for │
│ any purpose with or without fee is hereby granted, provided that the │
│ above copyright notice and this permission notice appear in all copies. │
│ │
│ THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL │
│ WARRANTIES WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED │
│ WARRANTIES OF MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE │
│ AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL │
│ DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR │
│ PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER │
│ TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR │
│ PERFORMANCE OF THIS SOFTWARE. │
╚─────────────────────────────────────────────────────────────────────────────*/
#include "ape/sections.internal.h"
#include "libc/dce.h"
#include "libc/intrin/bits.h"
#include "libc/runtime/internal.h"
#include "libc/runtime/morph.h"
#include "libc/thread/tls.h"
typedef char xmm_t __attribute__((__vector_size__(16), __aligned__(1)));
privileged void __morph_tls(void) {
// We need to rewrite SysV _Thread_local code. You MUST use the
// -mno-tls-direct-seg-refs flag which generates code like this
//
// 64 48 8b 0R4 25 00 00 00 00 mov %fs:0,%R
// 64 48 03 0R4 25 00 00 00 00 add %fs:0,%R
//
// Which on Mac we can replace with this:
//
// 65 48 8b 0R4 25 30 00 00 00 mov %gs:0x30,%R
//
// Since we have no idea where the TLS instructions exist in the
// binary, we need to disassemble the whole program image. This'll
// potentially take a few milliseconds for some larger programs.
//
// We check `_tls_content` which is generated by the linker script
// since it lets us determine ahead of time if _Thread_local vars
// have actually been linked into this program.
if ((intptr_t)_tls_content && (IsWindows() || IsXnu())) {
int n;
uint64_t w;
sigset_t mask;
unsigned m, dis;
unsigned char *p;
__morph_begin(&mask);
if (IsXnu()) {
// Apple is quite straightforward to patch. We basically
// just change the segment register, and the linear slot
// address 0x30 was promised to us, according to Go team
// https://github.com/golang/go/issues/23617
dis = 0x30;
} else {
// MSVC __declspec(thread) generates binary code for this
// %gs:0x1480 abi. So long as TlsAlloc() isn't called >64
// times we should be good.
dis = 0x1480 + __tls_index * 8;
}
// iterate over modifiable code looking for 9 byte instruction
// this would take 30 ms using xed to enable tls on python.com
for (p = _ereal; p + 9 <= __privileged_start; p += n) {
// use sse to zoom zoom to fs register prefixes
// that way it'll take 1 ms to morph python.com
while (p + 9 + 16 <= __privileged_start) {
if ((m = __builtin_ia32_pmovmskb128(
*(xmm_t *)p == (xmm_t){0144, 0144, 0144, 0144, 0144, 0144,
0144, 0144, 0144, 0144, 0144, 0144,
0144, 0144, 0144, 0144}))) {
m = __builtin_ctzll(m);
p += m;
break;
} else {
p += 16;
}
}
// we're checking for the following expression:
// 0144 == p[0] && // %fs
// 0110 == (p[1] & 0373) && // rex.w (and ignore rex.r)
// (0213 == p[2] || // mov reg/mem → reg (word-sized)
// 0003 == p[2]) && // add reg/mem → reg (word-sized)
// 0004 == (p[3] & 0307) && // mod/rm (4,reg,0) means sib → reg
// 0045 == p[4] && // sib (5,4,0) → (rbp,rsp,0) → disp32
// 0000 == p[5] && // displacement (von Neumann endian)
// 0000 == p[6] && // displacement
// 0000 == p[7] && // displacement
// 0000 == p[8] // displacement
w = READ64LE(p) & READ64LE("\377\373\377\307\377\377\377\377");
if ((w == READ64LE("\144\110\213\004\045\000\000\000") ||
w == READ64LE("\144\110\003\004\045\000\000\000")) &&
!p[8]) {
// now change the code
p[0] = 0145; // change %fs to %gs
p[5] = (dis & 0x000000ff) >> 000; // displacement
p[6] = (dis & 0x0000ff00) >> 010; // displacement
p[7] = (dis & 0x00ff0000) >> 020; // displacement
p[8] = (dis & 0xff000000) >> 030; // displacement
// advance to the next instruction
n = 9;
} else {
n = 1;
}
}
__morph_end(&mask);
}
}

2
libc/runtime/mprotect-nt.greg.c
View file

@ -16,8 +16,8 @@
│ TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR │
│ PERFORMANCE OF THIS SOFTWARE. │
╚─────────────────────────────────────────────────────────────────────────────*/
#include "libc/intrin/directmap.internal.h"
#include "libc/nt/memory.h"
#include "libc/runtime/directmap.internal.h"
#include "libc/runtime/internal.h"
#include "libc/runtime/memtrack.internal.h"

2
libc/runtime/mremap.c
View file

@ -21,12 +21,12 @@
#include "libc/dce.h"
#include "libc/intrin/asan.internal.h"
#include "libc/intrin/describeflags.internal.h"
#include "libc/intrin/directmap.internal.h"
#include "libc/intrin/likely.h"
#include "libc/intrin/strace.internal.h"
#include "libc/intrin/weaken.h"
#include "libc/macros.internal.h"
#include "libc/nt/runtime.h"
#include "libc/runtime/directmap.internal.h"
#include "libc/runtime/memtrack.internal.h"
#include "libc/runtime/runtime.h"
#include "libc/sysv/consts/map.h"

2
libc/runtime/munmap.c
View file

@ -22,13 +22,13 @@
#include "libc/dce.h"
#include "libc/errno.h"
#include "libc/intrin/asan.internal.h"
#include "libc/intrin/directmap.internal.h"
#include "libc/intrin/likely.h"
#include "libc/intrin/strace.internal.h"
#include "libc/log/backtrace.internal.h"
#include "libc/log/libfatal.internal.h"
#include "libc/log/log.h"
#include "libc/macros.internal.h"
#include "libc/runtime/directmap.internal.h"
#include "libc/runtime/internal.h"
#include "libc/runtime/memtrack.internal.h"
#include "libc/runtime/runtime.h"

8
libc/runtime/printargs.c
View file

@ -176,8 +176,8 @@ textstartup void __printargs(const char *prologue) {
if (!PLEDGED(STDIO)) return;
--__ftrace;
--__strace;
ftrace_enabled(-1);
strace_enabled(-1);
e = errno;
PRINT("");
@ -618,7 +618,7 @@ textstartup void __printargs(const char *prologue) {
}
PRINT("");
++__strace;
++__ftrace;
strace_enabled(+1);
ftrace_enabled(+1);
errno = e;
}

6
libc/runtime/runtime.h
View file

@ -16,8 +16,8 @@ extern char **__envp; /* CRT */
extern unsigned long *__auxv; /* CRT */
extern intptr_t __oldstack; /* CRT */
extern uint64_t __nosync; /* SYS */
extern _Atomic(int) __ftrace; /* SYS */
extern _Atomic(int) __strace; /* SYS */
extern int __strace; /* SYS */
extern int __ftrace; /* SYS */
extern char *program_invocation_name; /* RII */
extern char *program_invocation_short_name; /* RII */
extern uint64_t __syscount; /* RII */
@ -93,6 +93,8 @@ void _weakfree(void *);
void free_s(void *) paramsnonnull() libcesque;
int OpenExecutable(void);
int ftrace_install(void);
int ftrace_enabled(int);
int strace_enabled(int);
long GetResourceLimit(int);
long GetMaxFd(void);
char *GetProgramExecutableName(void);

59
libc/runtime/set_tls.c Normal file
View file

@ -0,0 +1,59 @@
/*-*- mode:c;indent-tabs-mode:nil;c-basic-offset:2;tab-width:8;coding:utf-8 -*-│
│vi: set net ft=c ts=2 sts=2 sw=2 fenc=utf-8 :vi│
╞══════════════════════════════════════════════════════════════════════════════╡
│ Copyright 2022 Justine Alexandra Roberts Tunney │
│ │
│ Permission to use, copy, modify, and/or distribute this software for │
│ any purpose with or without fee is hereby granted, provided that the │
│ above copyright notice and this permission notice appear in all copies. │
│ │
│ THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL │
│ WARRANTIES WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED │
│ WARRANTIES OF MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE │
│ AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL │
│ DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR │
│ PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER │
│ TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR │
│ PERFORMANCE OF THIS SOFTWARE. │
╚─────────────────────────────────────────────────────────────────────────────*/
#include "libc/assert.h"
#include "libc/calls/calls.h"
#include "libc/dce.h"
#include "libc/errno.h"
#include "libc/nexgen32e/msr.h"
#include "libc/nt/thread.h"
#include "libc/thread/tls.h"
int sys_set_tls();
void __set_tls(struct CosmoTib *tib) {
// ask the operating system to change the x86 segment register
int ax, dx;
if (IsWindows()) {
__tls_index = TlsAlloc();
_npassert(0 <= __tls_index && __tls_index < 64);
asm("mov\t%1,%%gs:%0" : "=m"(*((long *)0x1480 + __tls_index)) : "r"(tib));
} else if (IsFreebsd()) {
sys_set_tls(129 /*AMD64_SET_FSBASE*/, tib);
} else if (IsLinux()) {
sys_set_tls(ARCH_SET_FS, tib);
} else if (IsNetbsd()) {
// netbsd has sysarch(X86_SET_FSBASE) but we can't use that because
// signal handlers will cause it to be reset due to not setting the
// _mc_tlsbase field in struct mcontext_netbsd.
sys_set_tls(tib);
} else if (IsOpenbsd()) {
sys_set_tls(tib);
} else if (IsXnu()) {
// thread_fast_set_cthread_self has a weird ABI
int e = errno;
sys_set_tls((intptr_t)tib - 0x30);
errno = e;
} else {
uint64_t val = (uint64_t)tib;
asm volatile("wrmsr"
: /* no outputs */
: "c"(MSR_IA32_FS_BASE), "a"((uint32_t)val),
"d"((uint32_t)(val >> 32)));
}
}

2
libc/runtime/straceinit.greg.c
View file

@ -31,7 +31,7 @@ textstartup int __strace_init(int argc, char **argv, char **envp, long *auxv) {
/* asan isn't initialized yet at runlevel 300 */
if (__intercept_flag(&argc, argv, "--strace") ||
__atoul(nulltoempty(__getenv(envp, "STRACE")))) {
atomic_store_explicit(&__strace, 1, memory_order_relaxed);
strace_enabled(+1);
}
return (__argc = argc);
}

2
libc/runtime/symbols.c
View file

@ -24,7 +24,7 @@
#include "libc/runtime/symbols.internal.h"
void __init_symbols(void) {
if (__strace || (IsAsan() && _weaken(__die))) {
if (__strace > 0 || (IsAsan() && _weaken(__die))) {
GetSymbolTable();
}
}

21
libc/runtime/winmain.greg.c
View file

@ -19,27 +19,15 @@
#include "libc/calls/state.internal.h"
#include "libc/calls/syscall_support-nt.internal.h"
#include "libc/dce.h"
#include "libc/elf/pf2prot.internal.h"
#include "libc/errno.h"
#include "libc/fmt/fmt.h"
#include "libc/intrin/bits.h"
#include "libc/intrin/describeflags.internal.h"
#include "libc/intrin/nomultics.internal.h"
#include "libc/intrin/pushpop.h"
#include "libc/intrin/strace.internal.h"
#include "libc/intrin/weaken.h"
#include "libc/log/libfatal.internal.h"
#include "libc/macros.internal.h"
#include "libc/nexgen32e/nt2sysv.h"
#include "libc/nexgen32e/rdtsc.h"
#include "libc/nt/console.h"
#include "libc/nt/enum/consolemodeflags.h"
#include "libc/nt/enum/filemapflags.h"
#include "libc/nt/enum/filetype.h"
#include "libc/nt/enum/loadlibrarysearch.h"
#include "libc/nt/enum/pageflags.h"
#include "libc/nt/enum/version.h"
#include "libc/nt/files.h"
#include "libc/nt/memory.h"
#include "libc/nt/pedef.internal.h"
#include "libc/nt/process.h"
@ -47,16 +35,11 @@
#include "libc/nt/signals.h"
#include "libc/nt/struct/ntexceptionpointers.h"
#include "libc/nt/struct/teb.h"
#include "libc/nt/synchronization.h"
#include "libc/nt/thunk/msabi.h"
#include "libc/runtime/directmap.internal.h"
#include "libc/runtime/internal.h"
#include "libc/runtime/memtrack.internal.h"
#include "libc/runtime/runtime.h"
#include "libc/runtime/stack.h"
#include "libc/runtime/winargs.internal.h"
#include "libc/sock/internal.h"
#include "libc/str/str.h"
#include "libc/str/utf16.h"
#if IsTiny()
__msabi extern typeof(CreateFileMapping) *const __imp_CreateFileMappingW;
@ -79,7 +62,6 @@ __msabi extern typeof(VirtualProtect) *const __imp_VirtualProtect;
* TODO: How can we ensure we never overlap with KERNEL32.DLL?
*/
extern uint32_t __winmainpid;
extern int64_t __wincrashearly;
extern const char kConsoleHandles[3];
@ -169,7 +151,6 @@ __msabi static textwindows wontreturn void WinMainNew(const char16_t *cmdline) {
version = NtGetPeb()->OSMajorVersion;
__oldstack = (intptr_t)__builtin_frame_address(0);
if ((intptr_t)v_ntsubsystem == kNtImageSubsystemWindowsCui && version >= 10) {
__winmainpid = __pid;
rc = SetConsoleCP(kNtCpUtf8);
NTTRACE("SetConsoleCP(kNtCpUtf8) → %hhhd", rc);
rc = SetConsoleOutputCP(kNtCpUtf8);

2
libc/sock/recvmsg.c
View file

@ -93,7 +93,7 @@ ssize_t recvmsg(int fd, struct msghdr *msg, int flags) {
}
#if defined(SYSDEBUG) && _DATATRACE
if (__strace > 0) {
if (__strace > 0 && strace_enabled(0) > 0) {
if (!msg || (rc == -1 && errno == EFAULT)) {
DATATRACE("recvmsg(%d, %p, %#x) → %'ld% m", fd, msg, flags, rc);
} else {

2
libc/sock/sendmsg.c
View file

@ -80,7 +80,7 @@ ssize_t sendmsg(int fd, const struct msghdr *msg, int flags) {
}
#if defined(SYSDEBUG) && _DATATRACE
if (__strace > 0) {
if (__strace > 0 && strace_enabled(0) > 0) {
kprintf(STRACE_PROLOGUE "sendmsg(");
if ((!IsAsan() && kisdangerous(msg)) ||
(IsAsan() && !__asan_is_valid(msg, sizeof(*msg)))) {

4
libc/sock/struct/pollfd.internal.h
View file

@ -2,6 +2,7 @@
#define COSMOPOLITAN_LIBC_SOCK_STRUCT_POLLFD_INTERNAL_H_
#include "libc/calls/struct/sigset.h"
#include "libc/calls/struct/timespec.h"
#include "libc/mem/alloca.h"
#include "libc/sock/struct/pollfd.h"
#if !(__ASSEMBLER__ + __LINKER__ + 0)
COSMOPOLITAN_C_START_
@ -12,6 +13,9 @@ int sys_ppoll(struct pollfd *, size_t, const struct timespec *,
int sys_poll_metal(struct pollfd *, size_t, unsigned);
int sys_poll_nt(struct pollfd *, uint64_t, uint64_t *, const sigset_t *) hidden;
const char *DescribePollFds(char[300], ssize_t, struct pollfd *, size_t);
#define DescribePollFds(x, y, z) DescribePollFds(alloca(300), x, y, z)
COSMOPOLITAN_C_END_
#endif /* !(__ASSEMBLER__ + __LINKER__ + 0) */
#endif /* COSMOPOLITAN_LIBC_SOCK_STRUCT_POLLFD_INTERNAL_H_ */

2
libc/stdio/dirstream.c
View file

@ -263,7 +263,7 @@ DIR *opendir(const char *name) {
struct stat st;
struct Zipos *zip;
struct ZiposUri zipname;
if (!name || (IsAsan() && !__asan_is_valid(name, 1))) {
if (!name || (IsAsan() && !__asan_is_valid_str(name))) {
efault();
res = 0;
} else if (_weaken(__zipos_get) &&

2
libc/sysv/calls/sys_enable_tls.s
View file

@ -1,2 +0,0 @@
.include "o/libc/sysv/macros.internal.inc"
.scall sys_enable_tls,0x13d1490a5300309e,globl,hidden

Some files were not shown because too many files have changed in this diff Show more