mirror of
https://github.com/jart/cosmopolitan.git
synced 2025-02-12 09:17:53 +00:00
0cb6b6ff4b
Now that we have understandable system call tracing on Windows, this change rewrites many of the polyfill internals for that platform, to help things get closer to tip top shape. Support for complex forking scenarios had been in a regressed state for quite some time. Now, it works! Subsequent changes should be able to address the performance.
310 lines
11 KiB
C
310 lines
11 KiB
C
/*-*- mode:c;indent-tabs-mode:nil;c-basic-offset:2;tab-width:8;coding:utf-8 -*-│
|
|
│vi: set net ft=c ts=2 sts=2 sw=2 fenc=utf-8 :vi│
|
|
╞══════════════════════════════════════════════════════════════════════════════╡
|
|
│ Copyright 2020 Justine Alexandra Roberts Tunney │
|
|
│ │
|
|
│ Permission to use, copy, modify, and/or distribute this software for │
|
|
│ any purpose with or without fee is hereby granted, provided that the │
|
|
│ above copyright notice and this permission notice appear in all copies. │
|
|
│ │
|
|
│ THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL │
|
|
│ WARRANTIES WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED │
|
|
│ WARRANTIES OF MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE │
|
|
│ AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL │
|
|
│ DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR │
|
|
│ PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER │
|
|
│ TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR │
|
|
│ PERFORMANCE OF THIS SOFTWARE. │
|
|
╚─────────────────────────────────────────────────────────────────────────────*/
|
|
#include "libc/assert.h"
|
|
#include "libc/bits/likely.h"
|
|
#include "libc/bits/weaken.h"
|
|
#include "libc/calls/calls.h"
|
|
#include "libc/calls/internal.h"
|
|
#include "libc/calls/strace.internal.h"
|
|
#include "libc/dce.h"
|
|
#include "libc/errno.h"
|
|
#include "libc/intrin/asan.internal.h"
|
|
#include "libc/log/backtrace.internal.h"
|
|
#include "libc/log/libfatal.internal.h"
|
|
#include "libc/log/log.h"
|
|
#include "libc/macros.internal.h"
|
|
#include "libc/rand/rand.h"
|
|
#include "libc/runtime/directmap.internal.h"
|
|
#include "libc/runtime/memtrack.internal.h"
|
|
#include "libc/runtime/runtime.h"
|
|
#include "libc/str/str.h"
|
|
#include "libc/sysv/consts/map.h"
|
|
#include "libc/sysv/consts/prot.h"
|
|
#include "libc/sysv/errfuns.h"
|
|
|
|
#define IP(X) (intptr_t)(X)
|
|
#define VIP(X) (void *)IP(X)
|
|
#define ALIGNED(p) (!(IP(p) & (FRAMESIZE - 1)))
|
|
#define ADDR(x) ((int64_t)((uint64_t)(x) << 32) >> 16)
|
|
#define SHADE(x) (((intptr_t)(x) >> 3) + 0x7fff8000)
|
|
#define FRAME(x) ((int)((intptr_t)(x) >> 16))
|
|
|
|
forceinline wontreturn void Die(void) {
|
|
if (weaken(__die)) weaken(__die)();
|
|
abort();
|
|
}
|
|
|
|
noasan static bool IsMapped(char *p, size_t n) {
|
|
return OverlapsImageSpace(p, n) || IsMemtracked(FRAME(p), FRAME(p + (n - 1)));
|
|
}
|
|
|
|
noasan static bool NeedAutomap(char *p, size_t n) {
|
|
return !p || OverlapsArenaSpace(p, n) || OverlapsShadowSpace(p, n) ||
|
|
IsMapped(p, n);
|
|
}
|
|
|
|
noasan static bool ChooseMemoryInterval(int x, int n, int *res) {
|
|
int i;
|
|
if (_mmi.i) {
|
|
i = FindMemoryInterval(&_mmi, x);
|
|
if (i < _mmi.i) {
|
|
if (x + n < _mmi.p[i].x) {
|
|
*res = x;
|
|
return true;
|
|
}
|
|
while (++i < _mmi.i) {
|
|
if (_mmi.p[i].x - _mmi.p[i - 1].y > n) {
|
|
*res = _mmi.p[i - 1].y + 1;
|
|
return true;
|
|
}
|
|
}
|
|
}
|
|
if (INT_MAX - _mmi.p[i - 1].y >= n) {
|
|
*res = _mmi.p[i - 1].y + 1;
|
|
return true;
|
|
}
|
|
return false;
|
|
} else {
|
|
*res = x;
|
|
return true;
|
|
}
|
|
}
|
|
|
|
noasan static bool Automap(int n, int *res) {
|
|
*res = -1;
|
|
if (ChooseMemoryInterval(FRAME(kAutomapStart), n, res)) {
|
|
assert(*res >= FRAME(kAutomapStart));
|
|
if (*res + n <= FRAME(kAutomapStart + (kAutomapStart - 1))) {
|
|
return true;
|
|
} else {
|
|
STRACE("mmap(%.12p, %p) ENOMEM (automap interval exhausted)", ADDR(*res),
|
|
ADDR(n + 1));
|
|
return false;
|
|
}
|
|
} else {
|
|
STRACE("mmap(%.12p, %p) ENOMEM (automap failed)", ADDR(*res), ADDR(n + 1));
|
|
return false;
|
|
}
|
|
}
|
|
|
|
static noasan void *MapMemory(void *addr, size_t size, int prot, int flags,
|
|
int fd, int64_t off, int f, int x, int n) {
|
|
struct DirectMap dm;
|
|
dm = sys_mmap(addr, size, prot, f, fd, off);
|
|
if (UNLIKELY(dm.addr == MAP_FAILED)) {
|
|
if (IsWindows() && (flags & MAP_FIXED)) {
|
|
STRACE("mmap(%.12p, %'ld) → %m (%s)", addr, size,
|
|
"can't recover from MAP_FIXED errors on Windows");
|
|
assert(!"MapMemory() failed");
|
|
Die();
|
|
}
|
|
return MAP_FAILED;
|
|
}
|
|
if (UNLIKELY(dm.addr != addr)) {
|
|
STRACE("KERNEL DIDN'T RESPECT MAP_FIXED");
|
|
assert(!"MapMemory() failed");
|
|
Die();
|
|
}
|
|
if (!IsWindows() && (flags & MAP_FIXED)) {
|
|
if (UntrackMemoryIntervals(addr, size)) {
|
|
STRACE("FIXED UNTRACK FAILED %m");
|
|
assert(!"MapMemory() failed");
|
|
Die();
|
|
}
|
|
}
|
|
if (TrackMemoryInterval(&_mmi, x, x + (n - 1), dm.maphandle, prot, flags, off,
|
|
size)) {
|
|
if (sys_munmap(addr, n) == -1) {
|
|
STRACE("TRACK MUNMAP FAILED %m");
|
|
assert(!"MapMemory() failed");
|
|
Die();
|
|
}
|
|
return MAP_FAILED;
|
|
}
|
|
if (weaken(__asan_map_shadow) && !OverlapsShadowSpace(addr, size)) {
|
|
weaken(__asan_map_shadow)((intptr_t)addr, size);
|
|
}
|
|
return addr;
|
|
}
|
|
|
|
/**
|
|
* Maps memory from system, one frame at a time.
|
|
*
|
|
* This is useful on Windows since it allows us to partially unmap or
|
|
* punch holes into existing mappings.
|
|
*/
|
|
static textwindows dontinline noasan void *MapMemories(char *addr, size_t size,
|
|
int prot, int flags,
|
|
int fd, int64_t off,
|
|
int f, int x, size_t n) {
|
|
int64_t oi, sz;
|
|
struct DirectMap dm;
|
|
size_t i, m = (n - 1) * FRAMESIZE;
|
|
assert(m < size && m + FRAMESIZE >= size);
|
|
oi = fd == -1 ? 0 : off + m;
|
|
sz = size - m;
|
|
dm = sys_mmap(addr + m, sz, prot, f, fd, oi);
|
|
if (dm.addr == MAP_FAILED) {
|
|
STRACE("MapMemories(%.12p+%lx/%lx) %m", addr, m, size);
|
|
return MAP_FAILED;
|
|
}
|
|
if (TrackMemoryInterval(&_mmi, x + (n - 1), x + (n - 1), dm.maphandle, prot,
|
|
flags, oi, sz) == -1) {
|
|
STRACE("MapMemories(%.12p+%lx/%lx) unrecoverable failure #1 %m", addr, m,
|
|
size);
|
|
assert(!"MapMemories() failed");
|
|
Die();
|
|
}
|
|
for (i = 0; i < m; i += FRAMESIZE) {
|
|
oi = fd == -1 ? 0 : off + i;
|
|
sz = FRAMESIZE;
|
|
dm = sys_mmap(addr + i, sz, prot, f, fd, oi);
|
|
if (dm.addr == MAP_FAILED ||
|
|
TrackMemoryInterval(&_mmi, x + i / FRAMESIZE, x + i / FRAMESIZE,
|
|
dm.maphandle, prot, flags, oi, sz) == -1) {
|
|
STRACE("MapMemories(%p+%x/%x) unrecoverable failure #2 %m", addr, i,
|
|
size);
|
|
assert(!"MapMemories() failed");
|
|
Die();
|
|
}
|
|
}
|
|
if (weaken(__asan_map_shadow) && !OverlapsShadowSpace(addr, size)) {
|
|
weaken(__asan_map_shadow)((intptr_t)addr, size);
|
|
}
|
|
return addr;
|
|
}
|
|
|
|
/**
|
|
* Beseeches system for page-table entries, e.g.
|
|
*
|
|
* char *m;
|
|
* m = mmap(NULL, FRAMESIZE, PROT_READ | PROT_WRITE,
|
|
* MAP_PRIVATE | MAP_ANONYMOUS, -1, 0);
|
|
* munmap(m, FRAMESIZE);
|
|
*
|
|
* @param addr should be 0 to let your memory manager choose address;
|
|
* unless MAP_FIXED or MAP_FIXED_NOREPLACE are specified in flags
|
|
* in which case this function will do precicely as you ask, even
|
|
* if p=0 (in which you need -fno-delete-null-pointer-checks); it
|
|
* needs to be 64kb aligned because it's a wise choice that sadly
|
|
* needs to be made mandatory because of Windows although you can
|
|
* use __sys_mmap() to circumvent it on System Five in which case
|
|
* runtime support services, e.g. asan memory safety, could break
|
|
* @param size must be >0 and needn't be a multiple of FRAMESIZE, but
|
|
* will be rounded up to FRAMESIZE automatically if MAP_ANONYMOUS
|
|
* is specified
|
|
* @param prot can have PROT_READ/PROT_WRITE/PROT_EXEC/PROT_NONE/etc.
|
|
* @param flags can have MAP_ANONYMOUS, MAP_SHARED, MAP_PRIVATE, etc.
|
|
* @param fd is an open()'d file descriptor, whose contents shall be
|
|
* made available w/ automatic reading at the chosen address and
|
|
* must be -1 if MAP_ANONYMOUS is specified
|
|
* @param off specifies absolute byte index of fd's file for mapping,
|
|
* should be zero if MAP_ANONYMOUS is specified, and sadly needs
|
|
* to be 64kb aligned too
|
|
* @return virtual base address of new mapping, or MAP_FAILED w/ errno
|
|
*/
|
|
noasan void *mmap(void *addr, size_t size, int prot, int flags, int fd,
|
|
int64_t off) {
|
|
struct DirectMap dm;
|
|
int a, b, i, f, m, n, x;
|
|
char mode[8], *p = addr;
|
|
if (UNLIKELY(!size)) {
|
|
STRACE("mmap(%.12p, %'zu) EINVAL (size=0)", p, size);
|
|
return VIP(einval());
|
|
}
|
|
if (UNLIKELY(!IsLegalSize(size))) {
|
|
STRACE("mmap(%.12p, %'zu) EINVAL (size isn't 48-bit)", p, size);
|
|
return VIP(einval());
|
|
}
|
|
if (UNLIKELY(!IsLegalPointer(p))) {
|
|
STRACE("mmap(%.12p, %'zu) EINVAL (p isn't 48-bit)", p, size);
|
|
return VIP(einval());
|
|
}
|
|
if (UNLIKELY(!ALIGNED(p))) {
|
|
STRACE("mmap(%.12p, %'zu) EINVAL (p isn't 64kb aligned)", p, size);
|
|
return VIP(einval());
|
|
}
|
|
if (UNLIKELY(fd < -1)) {
|
|
STRACE("mmap(%.12p, %'zu, fd=%d) EBADF", p, size, fd);
|
|
return VIP(ebadf());
|
|
}
|
|
if (UNLIKELY(!((fd != -1) ^ !!(flags & MAP_ANONYMOUS)))) {
|
|
STRACE("mmap(%.12p, %'zu, %s, %d, %'ld) EINVAL (fd anonymous mismatch)", p,
|
|
size, DescribeMapping(prot, flags, mode), fd, off);
|
|
return VIP(einval());
|
|
}
|
|
if (UNLIKELY(!(!!(flags & MAP_PRIVATE) ^ !!(flags & MAP_SHARED)))) {
|
|
STRACE("mmap(%.12p, %'zu) EINVAL (MAP_SHARED ^ MAP_PRIVATE)", p, size);
|
|
return VIP(einval());
|
|
}
|
|
if (UNLIKELY(off < 0)) {
|
|
STRACE("mmap(%.12p, %'zu, off=%'ld) EINVAL (neg off)", p, size, off);
|
|
return VIP(einval());
|
|
}
|
|
if (UNLIKELY(INT64_MAX - size < off)) {
|
|
STRACE("mmap(%.12p, %'zu, off=%'ld) EINVAL (too large)", p, size, off);
|
|
return VIP(einval());
|
|
}
|
|
if (UNLIKELY(!ALIGNED(off))) {
|
|
STRACE("mmap(%.12p, %'zu) EINVAL (p isn't 64kb aligned)", p, size);
|
|
return VIP(einval());
|
|
}
|
|
if ((flags & MAP_FIXED_NOREPLACE) && IsMapped(p, size)) {
|
|
if (OverlapsImageSpace(p, size)) {
|
|
STRACE("mmap(%.12p, %'zu) EFAULT (overlaps image)", p, size);
|
|
} else {
|
|
STRACE("mmap(%.12p, %'zu) EFAULT (overlaps existing)", p, size);
|
|
}
|
|
return VIP(efault());
|
|
}
|
|
STRACE("mmap(%.12p, %'zu, %s, %d, %'ld)% m", p, size,
|
|
DescribeMapping(prot, flags, mode), fd, off);
|
|
if (fd == -1) {
|
|
size = ROUNDUP(size, FRAMESIZE);
|
|
if (IsWindows()) {
|
|
prot |= PROT_WRITE; /* kludge */
|
|
}
|
|
}
|
|
n = FRAME(size) + !!(size & (FRAMESIZE - 1));
|
|
f = (flags & ~MAP_FIXED_NOREPLACE) | MAP_FIXED;
|
|
if (flags & MAP_FIXED) {
|
|
x = FRAME(p);
|
|
if (IsWindows()) {
|
|
if (UntrackMemoryIntervals(p, size)) {
|
|
STRACE("FIXED UNTRACK FAILED %m");
|
|
assert(!"mmap() failed");
|
|
Die();
|
|
}
|
|
}
|
|
} else if (!NeedAutomap(p, size)) {
|
|
x = FRAME(p);
|
|
} else if (!Automap(n, &x)) {
|
|
return VIP(enomem());
|
|
}
|
|
p = (char *)ADDR(x);
|
|
if (IsOpenbsd() && (f & MAP_GROWSDOWN)) { /* openbsd:dubstack */
|
|
dm = sys_mmap(p, size, prot, f & ~MAP_GROWSDOWN, fd, off);
|
|
if (dm.addr == MAP_FAILED) return MAP_FAILED;
|
|
}
|
|
if (!IsWindows()) {
|
|
return MapMemory(p, size, prot, flags, fd, off, f, x, n);
|
|
} else {
|
|
return MapMemories(p, size, prot, flags, fd, off, f, x, n);
|
|
}
|
|
}
|