mirrors/cosmopolitan
1
0
Fork 0
mirror of https://github.com/jart/cosmopolitan.git synced 2025-06-27 14:58:30 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 1

Optimize memory layout

Browse source
This commit is contained in:
Justine Tunney 2022-09-12 04:19:32 -07:00
parent 0305194d98
commit b69f3d2488
No known key found for this signature in database
GPG key ID: BE714B4575D6E328
41 changed files with 383 additions and 347 deletions
Download patch file Download diff file Expand all files Collapse all files

64
libc/atomic.h
View file

@ -1,22 +1,8 @@
#ifndef COSMOPOLITAN_LIBC_ATOMIC_H_
#define COSMOPOLITAN_LIBC_ATOMIC_H_
#include "libc/inttypes.h"
#if !(__ASSEMBLER__ + __LINKER__ + 0)
COSMOPOLITAN_C_START_
/**
* @fileoverview C11 Atomic Types
*
* We supoprt C++ and old C compilers. It's recommended you use macros
* like `_Atomic(int)` rather than `_Atomic int` or `atomic_int` since
* we only define a portability macro for the syntax `_Atomic(T)`.
*
* @see libc/integral/c.inc
* @see libc/intrin/atomic.h
*/
#define atomic_bool _Atomic(_Bool)
#define atomic_bool32 atomic_int32
#define atomic_bool32 _Atomic(__INT32_TYPE__)
#define atomic_char _Atomic(char)
#define atomic_schar _Atomic(signed char)
#define atomic_uchar _Atomic(unsigned char)
@ -28,29 +14,29 @@ COSMOPOLITAN_C_START_
#define atomic_ulong _Atomic(unsigned long)
#define atomic_llong _Atomic(long long)
#define atomic_ullong _Atomic(unsigned long long)
#define atomic_char16_t _Atomic(char16_t)
#define atomic_char32_t _Atomic(char32_t)
#define atomic_wchar_t _Atomic(wchar_t)
#define atomic_intptr_t _Atomic(intptr_t)
#define atomic_uintptr_t _Atomic(uintptr_t)
#define atomic_size_t _Atomic(size_t)
#define atomic_ptrdiff_t _Atomic(ptrdiff_t)
#define atomic_int_fast8_t _Atomic(int_fast8_t)
#define atomic_uint_fast8_t _Atomic(uint_fast8_t)
#define atomic_int_fast16_t _Atomic(int_fast16_t)
#define atomic_uint_fast16_t _Atomic(uint_fast16_t)
#define atomic_int_fast32_t _Atomic(int_fast32_t)
#define atomic_uint_fast32_t _Atomic(uint_fast32_t)
#define atomic_int_fast64_t _Atomic(int_fast64_t)
#define atomic_uint_fast64_t _Atomic(uint_fast64_t)
#define atomic_int_least8_t _Atomic(int_least8_t)
#define atomic_uint_least8_t _Atomic(uint_least8_t)
#define atomic_int_least16_t _Atomic(int_least16_t)
#define atomic_uint_least16_t _Atomic(uint_least16_t)
#define atomic_int_least32_t _Atomic(int_least32_t)
#define atomic_uint_least32_t _Atomic(uint_least32_t)
#define atomic_int_least64_t _Atomic(int_least64_t)
#define atomic_uint_least64_t _Atomic(uint_least64_t)
#define atomic_char16_t _Atomic(__CHAR16_TYPE__)
#define atomic_char32_t _Atomic(__CHAR32_TYPE__)
#define atomic_wchar_t _Atomic(__WCHAR_TYPE__)
#define atomic_intptr_t _Atomic(__INTPTR_TYPE__)
#define atomic_uintptr_t _Atomic(__UINTPTR_TYPE__)
#define atomic_size_t _Atomic(__SIZE_TYPE__)
#define atomic_ptrdiff_t _Atomic(__PTRDIFF_TYPE__)
#define atomic_int_fast8_t _Atomic(__INT_FAST8_TYPE__)
#define atomic_uint_fast8_t _Atomic(__UINT_FAST8_TYPE__)
#define atomic_int_fast16_t _Atomic(__INT_FAST16_TYPE__)
#define atomic_uint_fast16_t _Atomic(__UINT_FAST16_TYPE__)
#define atomic_int_fast32_t _Atomic(__INT_FAST32_TYPE__)
#define atomic_uint_fast32_t _Atomic(__UINT_FAST32_TYPE__)
#define atomic_int_fast64_t _Atomic(__INT_FAST64_TYPE__)
#define atomic_uint_fast64_t _Atomic(__UINT_FAST64_TYPE__)
#define atomic_int_least8_t _Atomic(__INT_LEAST8_TYPE__)
#define atomic_uint_least8_t _Atomic(__UINT_LEAST8_TYPE__)
#define atomic_int_least16_t _Atomic(__INT_LEAST16_TYPE__)
#define atomic_uint_least16_t _Atomic(__UINT_LEAST16_TYPE__)
#define atomic_int_least32_t _Atomic(__INT_LEAST32_TYPE__)
#define atomic_uint_least32_t _Atomic(__UINT_LEAST32_TYPE__)
#define atomic_int_least64_t _Atomic(__INT_LEAST64_TYPE__)
#define atomic_uint_least64_t _Atomic(__UINT_LEAST64_TYPE__)
#ifdef __CLANG_ATOMIC_BOOL_LOCK_FREE
#define ATOMIC_BOOL_LOCK_FREE __CLANG_ATOMIC_BOOL_LOCK_FREE
@ -76,6 +62,4 @@ COSMOPOLITAN_C_START_
#define ATOMIC_POINTER_LOCK_FREE __GCC_ATOMIC_POINTER_LOCK_FREE
#endif
COSMOPOLITAN_C_END_
#endif /* !(__ASSEMBLER__ + __LINKER__ + 0) */
#endif /* COSMOPOLITAN_LIBC_ATOMIC_H_ */

4
libc/calls/munmap-sysv.c
View file

@ -38,7 +38,7 @@ int sys_munmap(void *p, size_t n) {
} else {
rc = sys_munmap_metal(p, n);
}
KERNTRACE("sys_munmap(%p%s, %'zu) → %d", p, DescribeFrame((intptr_t)p >> 16),
n, rc);
KERNTRACE("sys_munmap(%p /* %s */, %'zu) → %d", p,
DescribeFrame((intptr_t)p >> 16), n, rc);
return rc;
}

30
libc/calls/weirdtypes.h
View file

@ -37,34 +37,8 @@
#define uid_t uint32_t
#define rlim_t uint64_t /* int64_t on bsd */
typedef __INT_FAST8_TYPE__ int_fast8_t;
typedef __UINT_FAST8_TYPE__ uint_fast8_t;
typedef __INT_FAST16_TYPE__ int_fast16_t;
typedef __UINT_FAST16_TYPE__ uint_fast16_t;
typedef __INT_FAST32_TYPE__ int_fast32_t;
typedef __UINT_FAST32_TYPE__ uint_fast32_t;
typedef __INT_FAST64_TYPE__ int_fast64_t;
typedef __UINT_FAST64_TYPE__ uint_fast64_t;
#define TIME_T_MAX __INT64_MAX__
#define UINT_FAST64_MAX __UINT_FAST64_MAX__
#define UINT_FAST8_MAX __UINT_FAST8_MAX__
#define INT_FAST32_MAX __INT_FAST32_MAX__
#define INT_FAST16_MAX __INT_FAST16_MAX__
#define UINT_FAST32_MAX __UINT_FAST32_MAX__
#define INT_FAST8_MAX __INT_FAST8_MAX__
#define INT_FAST64_MAX __INT_FAST64_MAX__
#define UINT_FAST16_MAX __UINT_FAST16_MAX__
#define TIME_T_MIN (-TIME_T_MAX - 1)
#define UINT_FAST64_MIN (-UINT_FAST64_MAX - 1)
#define UINT_FAST8_MIN (-UINT_FAST8_MAX - 1)
#define INT_FAST32_MIN (-INT_FAST32_MAX - 1)
#define INT_FAST16_MIN (-INT_FAST16_MAX - 1)
#define UINT_FAST32_MIN (-UINT_FAST32_MAX - 1)
#define INT_FAST8_MIN (-INT_FAST8_MAX - 1)
#define INT_FAST64_MIN (-INT_FAST64_MAX - 1)
#define UINT_FAST16_MIN (-UINT_FAST16_MAX - 1)
#define TIME_T_MAX __INT64_MAX__
#define TIME_T_MIN (-TIME_T_MAX - 1)
#endif /* !(__ASSEMBLER__ + __LINKER__ + 0) */
#endif /* COSMOPOLITAN_LIBC_CALLS_WEIRDTYPES_H_ */

54
libc/intrin/describeframe.c
View file

@ -16,32 +16,62 @@
│ TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR │
│ PERFORMANCE OF THIS SOFTWARE. │
╚─────────────────────────────────────────────────────────────────────────────*/
#include "libc/dce.h"
#include "libc/intrin/describeflags.internal.h"
#include "libc/intrin/kprintf.h"
#include "libc/macros.internal.h"
#include "libc/runtime/memtrack.internal.h"
#include "libc/runtime/runtime.h"
#include "libc/runtime/winargs.internal.h"
#define ADDR(x) ((int64_t)((uint64_t)(x) << 32) >> 16)
#define UNSHADOW(x) ((int64_t)(MAX(0, (x)-0x7fff8000)) << 3)
#define FRAME(x) ((int)((x) >> 16))
const char *(DescribeFrame)(char buf[32], int x) {
/* asan runtime depends on this function */
char *p;
if (IsShadowFrame(x)) {
ksnprintf(buf, 32, " shadow=%.8x", FRAME(UNSHADOW(ADDR(x))));
return buf;
return " shadow ";
static const char *GetFrameName(int x) {
if (!x) {
return "null";
} else if (IsShadowFrame(x)) {
return "shadow";
} else if (IsAutoFrame(x)) {
return " automap";
return "automap";
} else if (IsFixedFrame(x)) {
return " fixed ";
return "fixed";
} else if (IsArenaFrame(x)) {
return " arena ";
return "arena";
} else if (IsStaticStackFrame(x)) {
return " stack ";
return "stack";
} else if (IsGfdsFrame(x)) {
return "g_fds";
} else if (IsZiposFrame(x)) {
return "zipos";
} else if (IsNsyncFrame(x)) {
return "nsync";
} else if (IsMemtrackFrame(x)) {
return "memtrack";
} else if (IsOldStackFrame(x)) {
return "oldstack";
} else if (IsWindows() &&
(((GetStaticStackAddr(0) + GetStackSize()) >> 16) <= x &&
x <= ((GetStaticStackAddr(0) + GetStackSize() +
sizeof(struct WinArgs) - 1) >>
16))) {
return "winargs";
} else if ((int)((intptr_t)_base >> 16) <= x &&
x <= (int)(((intptr_t)_end - 1) >> 16)) {
return "image";
} else {
return "";
return "unknown";
}
}
const char *(DescribeFrame)(char buf[32], int x) {
char *p;
if (IsShadowFrame(x)) {
ksnprintf(buf, 64, "%s %s %.8x", GetFrameName(x),
GetFrameName(FRAME(UNSHADOW(ADDR(x)))), FRAME(UNSHADOW(ADDR(x))));
return buf;
} else {
return GetFrameName(x);
}
}

4
libc/intrin/describemapping.c
View file

@ -49,7 +49,7 @@ const char *(DescribeMapping)(char p[8], int prot, int flags) {
DescribeProt(p, prot);
p[3] = DescribeMapType(flags);
p[4] = (flags & MAP_ANONYMOUS) ? 'a' : '-';
p[6] = (flags & MAP_FIXED) ? 'F' : '-';
p[7] = 0;
p[5] = (flags & MAP_FIXED) ? 'F' : '-';
p[6] = 0;
return p;
}

7
libc/intrin/directmap.c
View file

@ -46,8 +46,9 @@ struct DirectMap sys_mmap(void *addr, size_t size, int prot, int flags, int fd,
} else {
d = sys_mmap_nt(addr, size, prot, flags, fd, off);
}
KERNTRACE("sys_mmap(%.12p%s, %'zu, %s, %s, %d, %'ld) → {%.12p, %p}% m", addr,
DescribeFrame((intptr_t)addr >> 16), size, DescribeProtFlags(prot),
DescribeMapFlags(flags), fd, off, d.addr, d.maphandle);
KERNTRACE("sys_mmap(%.12p /* %s */, %'zu, %s, %s, %d, %'ld) → {%.12p, %p}% m",
addr, DescribeFrame((intptr_t)addr >> 16), size,
DescribeProtFlags(prot), DescribeMapFlags(flags), fd, off, d.addr,
d.maphandle);
return d;
}

1
libc/intrin/isatleastwindows10.greg.c
View file

@ -18,7 +18,6 @@
╚─────────────────────────────────────────────────────────────────────────────*/
#include "libc/assert.h"
#include "libc/dce.h"
#include "libc/nt/enum/version.h"
#include "libc/nt/version.h"
/**

62
libc/intrin/memtrack.greg.c
View file

@ -37,11 +37,20 @@
#include "libc/sysv/consts/prot.h"
#include "libc/sysv/errfuns.h"
#if IsModeDbg()
#define ASSERT_MEMTRACK() \
if (!AreMemoryIntervalsOk(mm)) { \
PrintMemoryIntervals(2, mm); \
notpossible; \
}
#else
#define ASSERT_MEMTRACK()
#endif
static void *MoveMemoryIntervals(struct MemoryInterval *d,
const struct MemoryInterval *s, int n) {
// asan runtime depends on this function
int i;
assert(n >= 0);
if (n < 0) unreachable;
if (d > s) {
for (i = n; i--;) {
d[i] = s[i];
@ -55,9 +64,8 @@ static void *MoveMemoryIntervals(struct MemoryInterval *d,
}
static void RemoveMemoryIntervals(struct MemoryIntervals *mm, int i, int n) {
// asan runtime depends on this function
assert(i >= 0);
assert(i + n <= mm->i);
if (i < 0) unreachable;
if (i + n > mm->i) unreachable;
MoveMemoryIntervals(mm->p + i, mm->p + i + n, mm->i - (i + n));
mm->i -= n;
}
@ -96,19 +104,14 @@ static bool ExtendMemoryIntervals(struct MemoryIntervals *mm) {
if (!dm.addr) return false;
mm->n = (size + gran) / sizeof(*mm->p);
}
#if IsModeDbg()
assert(AreMemoryIntervalsOk(mm));
#endif
ASSERT_MEMTRACK();
return true;
}
int CreateMemoryInterval(struct MemoryIntervals *mm, int i) {
// asan runtime depends on this function
int rc;
rc = 0;
assert(i >= 0);
assert(i <= mm->i);
assert(mm->n >= 0);
if (i < 0) unreachable;
if (i > mm->i) unreachable;
if (mm->n < 0) unreachable;
if (UNLIKELY(mm->i == mm->n) && !ExtendMemoryIntervals(mm)) return enomem();
MoveMemoryIntervals(mm->p + i + 1, mm->p + i, mm->i++ - i);
return 0;
@ -126,12 +129,9 @@ static int PunchHole(struct MemoryIntervals *mm, int x, int y, int i) {
int ReleaseMemoryIntervals(struct MemoryIntervals *mm, int x, int y,
void wf(struct MemoryIntervals *, int, int)) {
unsigned l, r;
#if IsModeDbg()
assert(y >= x);
assert(AreMemoryIntervalsOk(mm));
#endif
ASSERT_MEMTRACK();
if (y < x) unreachable;
if (!mm->i) return 0;
// binary search for the lefthand side
l = FindMemoryInterval(mm, x);
if (l == mm->i) return 0;
@ -140,8 +140,8 @@ int ReleaseMemoryIntervals(struct MemoryIntervals *mm, int x, int y,
// binary search for the righthand side
r = FindMemoryInterval(mm, y);
if (r == mm->i || (r > l && y < mm->p[r].x)) --r;
assert(r >= l);
assert(x <= mm->p[r].y);
if (r < l) unreachable;
if (x > mm->p[r].y) unreachable;
// remove the middle of an existing map
//
@ -162,11 +162,11 @@ int ReleaseMemoryIntervals(struct MemoryIntervals *mm, int x, int y,
// ----|mmmm|----------------- after
//
if (x > mm->p[l].x && x <= mm->p[l].y) {
assert(y >= mm->p[l].y);
if (y < mm->p[l].y) unreachable;
if (IsWindows()) return einval();
mm->p[l].size -= (size_t)(mm->p[l].y - (x - 1)) * FRAMESIZE;
mm->p[l].y = x - 1;
assert(mm->p[l].x <= mm->p[l].y);
if (mm->p[l].x > mm->p[l].y) unreachable;
++l;
}
@ -177,11 +177,11 @@ int ReleaseMemoryIntervals(struct MemoryIntervals *mm, int x, int y,
// ---------------|mm|-------- after
//
if (y >= mm->p[r].x && y < mm->p[r].y) {
assert(x <= mm->p[r].x);
if (x > mm->p[r].x) unreachable;
if (IsWindows()) return einval();
mm->p[r].size -= (size_t)((y + 1) - mm->p[r].x) * FRAMESIZE;
mm->p[r].x = y + 1;
assert(mm->p[r].x <= mm->p[r].y);
if (mm->p[r].x > mm->p[r].y) unreachable;
--r;
}
@ -197,16 +197,9 @@ int ReleaseMemoryIntervals(struct MemoryIntervals *mm, int x, int y,
int TrackMemoryInterval(struct MemoryIntervals *mm, int x, int y, long h,
int prot, int flags, bool readonlyfile, bool iscow,
long offset, long size) {
// asan runtime depends on this function
unsigned i;
#if IsModeDbg()
assert(y >= x);
if (!AreMemoryIntervalsOk(mm)) {
PrintMemoryIntervals(2, mm);
}
assert(AreMemoryIntervalsOk(mm));
#endif
ASSERT_MEMTRACK();
if (y < x) unreachable;
i = FindMemoryInterval(mm, x);
// try to extend the righthand side of the lefthand entry
@ -249,5 +242,6 @@ int TrackMemoryInterval(struct MemoryIntervals *mm, int x, int y, long h,
mm->p[i].iscow = iscow;
mm->p[i].readonlyfile = readonlyfile;
}
return 0;
}

20
libc/intrin/printmemoryintervals.c
View file

@ -16,6 +16,7 @@
│ TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR │
│ PERFORMANCE OF THIS SOFTWARE. │
╚─────────────────────────────────────────────────────────────────────────────*/
#include "libc/fmt/conv.h"
#include "libc/fmt/itoa.h"
#include "libc/intrin/describeflags.internal.h"
#include "libc/intrin/kprintf.h"
@ -34,28 +35,26 @@ static bool IsNoteworthyHole(unsigned i, const struct MemoryIntervals *mm) {
}
void PrintMemoryIntervals(int fd, const struct MemoryIntervals *mm) {
char *p, mappingbuf[8], framebuf[32];
long i, w, frames, maptally = 0, gaptally = 0;
char *p, mappingbuf[8], framebuf[64], sb[16];
long i, w, frames, maptally = 0;
for (w = i = 0; i < mm->i; ++i) {
w = MAX(w, LengthInt64Thousands(mm->p[i].y + 1 - mm->p[i].x));
}
for (i = 0; i < mm->i; ++i) {
frames = mm->p[i].y + 1 - mm->p[i].x;
maptally += frames;
kprintf("%08x-%08x %s %'*ldx%s", mm->p[i].x, mm->p[i].y,
kprintf("%08x-%08x %s %'*ldx %s", mm->p[i].x, mm->p[i].y,
(DescribeMapping)(mappingbuf, mm->p[i].prot, mm->p[i].flags), w,
frames, (DescribeFrame)(framebuf, mm->p[i].x));
if (mm->p[i].iscow) kprintf(" cow");
if (mm->p[i].readonlyfile) kprintf(" readonlyfile");
if (mm->p[i].size !=
(size_t)(mm->p[i].y - mm->p[i].x) * FRAMESIZE + FRAMESIZE) {
kprintf(" size=%'zu", mm->p[i].size);
}
sizefmt(sb, mm->p[i].size, 1024);
kprintf(" %sB", sb);
if (i + 1 < mm->i) {
frames = mm->p[i + 1].x - mm->p[i].y - 1;
if (frames && IsNoteworthyHole(i, mm)) {
gaptally += frames;
kprintf(" w/ %'ld frame hole", frames);
sizefmt(sb, frames * FRAMESIZE, 1024);
kprintf(" w/ %sB hole", sb);
}
}
if (mm->p[i].h != -1) {
@ -63,5 +62,6 @@ void PrintMemoryIntervals(int fd, const struct MemoryIntervals *mm) {
}
kprintf("\n");
}
kprintf("# %ld frames mapped w/ %'ld frames gapped\n", maptally, gaptally);
sizefmt(sb, maptally * FRAMESIZE, 1024);
kprintf("# %sB total mapped memory\n", sb);
}

9
libc/inttypes.h
View file

@ -10,6 +10,15 @@ typedef __UINT_LEAST32_TYPE__ uint_least32_t;
typedef __INT_LEAST64_TYPE__ int_least64_t;
typedef __UINT_LEAST64_TYPE__ uint_least64_t;
typedef __INT_FAST8_TYPE__ int_fast8_t;
typedef __UINT_FAST8_TYPE__ uint_fast8_t;
typedef __INT_FAST16_TYPE__ int_fast16_t;
typedef __UINT_FAST16_TYPE__ uint_fast16_t;
typedef __INT_FAST32_TYPE__ int_fast32_t;
typedef __UINT_FAST32_TYPE__ uint_fast32_t;
typedef __INT_FAST64_TYPE__ int_fast64_t;
typedef __UINT_FAST64_TYPE__ uint_fast64_t;
/*───────────────────────────────────────────────────────────────────────────│─╗
│ cosmopolitan § dismal format notation ─╬─│┼
╚────────────────────────────────────────────────────────────────────────────│*/

19
libc/limits.h
View file

@ -36,7 +36,6 @@
#define INTMAX_MAX __INTMAX_MAX__
#define UINTMAX_MAX __UINTMAX_MAX__
extern int __got_long_min;
#define SCHAR_MIN (-SCHAR_MAX - 1)
#define SHRT_MIN (-SHRT_MAX - 1)
#define INT_MIN (-INT_MAX - 1)
@ -108,4 +107,22 @@ extern int __got_long_min;
#define NL_SETMAX 255
#define NL_TEXTMAX 2048
#define UINT_FAST64_MAX __UINT_FAST64_MAX__
#define UINT_FAST8_MAX __UINT_FAST8_MAX__
#define INT_FAST32_MAX __INT_FAST32_MAX__
#define INT_FAST16_MAX __INT_FAST16_MAX__
#define UINT_FAST32_MAX __UINT_FAST32_MAX__
#define INT_FAST8_MAX __INT_FAST8_MAX__
#define INT_FAST64_MAX __INT_FAST64_MAX__
#define UINT_FAST16_MAX __UINT_FAST16_MAX__
#define UINT_FAST64_MIN (-UINT_FAST64_MAX - 1)
#define UINT_FAST8_MIN (-UINT_FAST8_MAX - 1)
#define INT_FAST32_MIN (-INT_FAST32_MAX - 1)
#define INT_FAST16_MIN (-INT_FAST16_MAX - 1)
#define UINT_FAST32_MIN (-UINT_FAST32_MAX - 1)
#define INT_FAST8_MIN (-INT_FAST8_MAX - 1)
#define INT_FAST64_MIN (-INT_FAST64_MAX - 1)
#define UINT_FAST16_MIN (-UINT_FAST16_MAX - 1)
#endif /* COSMOPOLITAN_LIBC_LIMITS_H_ */

1
libc/log/backtrace.internal.h
View file

@ -10,7 +10,6 @@ forceinline pureconst bool IsValidStackFramePointer(struct StackFrame *x) {
/* assumes __mmi_lock() is held */
return IsLegalPointer(x) && !((uintptr_t)x & 15) &&
(IsStaticStackFrame((uintptr_t)x >> 16) ||
IsSigAltStackFrame((uintptr_t)x >> 16) ||
IsOldStackFrame((uintptr_t)x >> 16) ||
/* lua coroutines need this */
IsMemtracked((uintptr_t)x >> 16, (uintptr_t)x >> 16));

8
libc/mem/arena.c
View file

@ -17,10 +17,10 @@
│ PERFORMANCE OF THIS SOFTWARE. │
╚─────────────────────────────────────────────────────────────────────────────*/
#include "libc/assert.h"
#include "libc/intrin/likely.h"
#include "libc/intrin/weaken.h"
#include "libc/calls/calls.h"
#include "libc/dce.h"
#include "libc/intrin/likely.h"
#include "libc/intrin/weaken.h"
#include "libc/limits.h"
#include "libc/log/log.h"
#include "libc/macros.internal.h"
@ -35,8 +35,8 @@
#include "libc/sysv/consts/prot.h"
#include "libc/sysv/errfuns.h"
#define BASE 0x50000000
#define SIZE 0x2ffe0000
#define BASE 0x50040000
#define SIZE 0x2ff80000
#define P(i) ((void *)(intptr_t)(i))
#define EXCHANGE(HOOK, SLOT) \
__arena_hook((intptr_t *)weaken(HOOK), (intptr_t *)(&(SLOT)))

7
libc/mem/pvalloc.c
View file

@ -20,7 +20,10 @@
#include "libc/mem/mem.h"
/**
* Equivalent to memalign(PAGESIZE, ROUNDUP(n, PAGESIZE)).
* Allocates granular aligned memory of granular size, i.e.
*
* memalign(sysconf(_SC_PAGESIZE),
* ROUNDUP(n, sysconf(_SC_PAGESIZE)));
*
* @param n number of bytes needed
* @return memory address, or NULL w/ errno
@ -28,5 +31,5 @@
* @threadsafe
*/
void *pvalloc(size_t n) {
return memalign(PAGESIZE, ROUNDUP(n, PAGESIZE));
return memalign(FRAMESIZE, ROUNDUP(n, FRAMESIZE));
}

6
libc/mem/valloc.c
View file

@ -19,7 +19,9 @@
#include "libc/mem/mem.h"
/**
* Equivalent to memalign(PAGESIZE, n).
* Allocates granular aligned memory, i.e.
*
* memalign(sysconf(_SC_PAGESIZE), n);
*
* @param n number of bytes needed
* @return memory address, or NULL w/ errno
@ -27,5 +29,5 @@
* @threadsafe
*/
void *valloc(size_t n) {
return memalign(PAGESIZE, n);
return memalign(FRAMESIZE, n);
}

2
libc/nt/version.h
View file

@ -7,7 +7,7 @@ COSMOPOLITAN_C_START_
bool IsAtLeastWindows10(void) pureconst;
bool32 GetVersionEx(struct NtOsVersionInfo *lpVersionInformation);
#if defined(__GCC_ASM_FLAG_OUTPUTS__) && !defined(__STRICT_ANSI__)
#if defined(__GNUC__) && !defined(__STRICT_ANSI__)
#define IsAtLeastWindows10() (GetNtMajorVersion() >= 10)
#define GetNtMajorVersion() \
({ \

4
libc/runtime/brk.c
View file

@ -31,8 +31,8 @@ uintptr_t __break;
*
* This can be used to allocate and deallocate memory. It won't
* conflict with malloc() and mmap(NULL, ...) allocations since
* APE binaries load the image at 0x400000 and does allocations
* starting at 0x100080000000. You should consult _end, or call
* APE binaries load the image at 0x440000 and does allocations
* starting at 0x100080040000. You should consult _end, or call
* sbrk(NULL), to figure out where the existing break is first.
*
* @return 0 on success or -1 w/ errno

82
libc/runtime/memtrack.internal.h
View file

@ -1,42 +1,30 @@
#ifndef COSMOPOLITAN_LIBC_RUNTIME_MEMTRACK_H_
#define COSMOPOLITAN_LIBC_RUNTIME_MEMTRACK_H_
#include "libc/assert.h"
#include "libc/intrin/midpoint.h"
#include "libc/dce.h"
#include "libc/intrin/midpoint.h"
#include "libc/intrin/nopl.internal.h"
#include "libc/macros.internal.h"
#include "libc/thread/tls.h"
#include "libc/nt/version.h"
#include "libc/runtime/stack.h"
#include "libc/sysv/consts/ss.h"
#include "libc/thread/tls.h"
#if !(__ASSEMBLER__ + __LINKER__ + 0)
COSMOPOLITAN_C_START_
#define kAutomapStart _kMem(0x100080000000, 0x000010000000)
#define kAutomapSize \
_kMem(0x200000000000 - 0x100080000000 - _kMmi(0x800000000000), \
0x000040000000 - 0x000010000000 - _kMmi(0x000080000000))
#define kMemtrackStart \
(ROUNDDOWN(_kMem(0x200000000000 - _kMmi(0x800000000000), \
0x000040000000 - _kMmi(0x000080000000)), \
FRAMESIZE * 8) - \
0x8000 * 8 /* so frame aligned after adding 0x7fff8000 */)
#define kMemtrackSize _kMem(_kMmi(0x800000000000), _kMmi(0x000080000000))
#define kMemtrackGran (!IsAsan() ? FRAMESIZE : FRAMESIZE * 8)
#define kFixedmapStart _kMem(0x300000000000, 0x000040000000)
#define kFixedmapSize \
_kMem(0x400000000000 - 0x300000000000, 0x000070000000 - 0x000040000000)
#define kMemtrackFdsStart \
(ROUNDDOWN(_kMem(0x6fe000000000, 0x68000000), FRAMESIZE * 8) - 0x8000 * 8)
#define kMemtrackFdsSize _kMem(0x001000000000, 0x04000000)
#define kMemtrackZiposStart \
(ROUNDDOWN(_kMem(0x6fd000000000, 0x6c000000), FRAMESIZE * 8) - 0x8000 * 8)
#define kMemtrackZiposSize _kMem(0x001000000000, 0x10000000)
#define _kMmi(VSPACE) \
ROUNDUP(VSPACE / FRAMESIZE * (intptr_t)sizeof(struct MemoryInterval), \
FRAMESIZE)
#define _kMem(NORMAL, WIN7) \
(!IsWindows() || IsAtLeastWindows10() ? NORMAL : WIN7)
#define kAutomapStart 0x100080040000
#define kAutomapSize (kMemtrackStart - kAutomapStart)
#define kMemtrackStart 0x1fe7fffc0000
#define kMemtrackSize (0x1ffffffc0000 - kMemtrackStart)
#define kFixedmapStart 0x300000040000
#define kFixedmapSize (0x400000040000 - kFixedmapStart)
#define kMemtrackFdsStart 0x6fe000040000
#define kMemtrackFdsSize (0x6feffffc0000 - kMemtrackFdsStart)
#define kMemtrackZiposStart 0x6fd000040000
#define kMemtrackZiposSize (0x6fdffffc0000 - kMemtrackZiposStart)
#define kMemtrackNsyncStart 0x6fc000040000
#define kMemtrackNsyncSize (0x6fcffffc0000 - kMemtrackNsyncStart)
#define kMemtrackGran (!IsAsan() ? FRAMESIZE : FRAMESIZE * 8)
struct MemoryInterval {
int x;
@ -88,23 +76,38 @@ forceinline pureconst bool IsLegalSize(size_t n) {
}
forceinline pureconst bool IsAutoFrame(int x) {
return (kAutomapStart >> 16) <= x &&
x <= ((kAutomapStart + (kAutomapSize - 1)) >> 16);
return (int)(kAutomapStart >> 16) <= x &&
x <= (int)((kAutomapStart + kAutomapSize - 1) >> 16);
}
forceinline pureconst bool IsMemtrackFrame(int x) {
return (kAutomapStart >> 16) <= x &&
x <= ((kAutomapStart + (kAutomapSize - 1)) >> 16);
return (int)(kAutomapStart >> 16) <= x &&
x <= (int)((kAutomapStart + kAutomapSize - 1) >> 16);
}
forceinline pureconst bool IsArenaFrame(int x) {
return 0x5000 <= x && x < 0x7ffe;
forceinline pureconst bool IsGfdsFrame(int x) {
return (int)(kMemtrackFdsStart >> 16) <= x &&
x <= (int)((kMemtrackFdsStart + kMemtrackFdsSize - 1) >> 16);
}
forceinline pureconst bool IsZiposFrame(int x) {
return (int)(kMemtrackZiposStart >> 16) <= x &&
x <= (int)((kMemtrackZiposStart + kMemtrackZiposSize - 1) >> 16);
}
forceinline pureconst bool IsNsyncFrame(int x) {
return (int)(kMemtrackNsyncStart >> 16) <= x &&
x <= (int)((kMemtrackNsyncStart + kMemtrackNsyncSize - 1) >> 16);
}
forceinline pureconst bool IsShadowFrame(int x) {
return 0x7fff <= x && x < 0x10008000;
}
forceinline pureconst bool IsArenaFrame(int x) {
return 0x5004 <= x && x <= 0x7ffb;
}
forceinline pureconst bool IsKernelFrame(int x) {
intptr_t stack = GetStaticStackAddr(0);
return (int)(stack >> 16) <= x &&
@ -123,15 +126,12 @@ forceinline pureconst bool IsStackFrame(int x) {
x <= (int)((stack + (GetStackSize() - FRAMESIZE)) >> 16);
}
forceinline pureconst bool IsSigAltStackFrame(int x) {
intptr_t stack = GetStackAddr();
return (int)(stack >> 16) <= x &&
x <= (int)((stack + (SIGSTKSZ - FRAMESIZE)) >> 16);
}
forceinline pureconst bool IsOldStackFrame(int x) {
intptr_t old = ROUNDDOWN(__oldstack, STACKSIZE);
return (old >> 16) <= x && x <= ((old + (STACKSIZE - FRAMESIZE)) >> 16);
/* openbsd uses 4mb stack by default */
/* freebsd uses 512mb stack by default */
/* most systems use 8mb stack by default */
intptr_t old = ROUNDDOWN(__oldstack, GetStackSize());
return (old >> 16) <= x && x <= ((old + (GetStackSize() - FRAMESIZE)) >> 16);
}
forceinline pureconst bool IsFixedFrame(int x) {

36
libc/runtime/memtrack64.txt
View file

@ -3,8 +3,8 @@
00000000-0000001f 2048kb null
00000020-0000003f 2048kb loader
00000040-00004fff 1276mb image
00005000-00007ffd 768mb arena
00000040-00004ffb 1276mb image
00005004-00007ffb 768mb arena
00007ffe-00007ffe 64kb free
# address sanitizer shadow memory
@ -264,10 +264,10 @@
0fd00000-0fdfffff 64gb asan
0fe00000-0fefffff 64gb asan
0ff00000-0fffffff 64gb asan
10000000-10007fff 2048mb asan
10000000-10008003 2048mb asan
# memory dedicated to mmap(NULL, ...) automation, e.g. malloc()
10008000-100fffff 62gb automap
10008004-100fffff 62gb automap
10100000-101fffff 64gb automap
10200000-102fffff 64gb automap
10300000-103fffff 64gb automap
@ -521,13 +521,13 @@
1fb00000-1fbfffff 64gb automap
1fc00000-1fcfffff 64gb automap
1fd00000-1fdfffff 64gb automap
1fe00000-1fe7ffff 32gb automap
1fe00000-1fe7fffb 32gb automap
1fe7fffc-1fefffff 32gb _mmi
1ff00000-1ffffffb 64gb _mmi
1ffffffc-1fffffff 256kb free
1ffffffc-20000003 256kb free
20000000-200fffff 64gb free
20000004-200fffff 64gb free
20100000-201fffff 64gb free
20200000-202fffff 64gb free
20300000-203fffff 64gb free
@ -782,10 +782,10 @@
2fc00000-2fcfffff 64gb free
2fd00000-2fdfffff 64gb free
2fe00000-2fefffff 64gb free
2ff00000-2fffffff 64gb free
2ff00000-30000003 64gb free
# memory recommended for application MAP_FIXED usage
30000000-300fffff 64gb fixedmap
30000004-300fffff 64gb fixedmap
30100000-301fffff 64gb fixedmap
30200000-302fffff 64gb fixedmap
30300000-303fffff 64gb fixedmap
@ -1040,9 +1040,9 @@
3fc00000-3fcfffff 64gb fixedmap
3fd00000-3fdfffff 64gb fixedmap
3fe00000-3fefffff 64gb fixedmap
3ff00000-3fffffff 64gb fixedmap
3ff00000-40000003 64gb fixedmap
40000000-400fffff 64gb free
40000004-400fffff 64gb free
40100000-401fffff 64gb free
40200000-402fffff 64gb free
40300000-403fffff 64gb free
@ -1808,14 +1808,14 @@
6f900000-6f9fffff 64gb free
6fa00000-6fafffff 64gb free
6fb00000-6fbfffff 64gb free
6fc00004-6fcfffff 64gb nsync
6fd00000-6fdfffff 64gb zipos
6fe00004-6feffffc 64gb g_fds
6ff00000-6ffffffd 64gb free
6ffffffe-6fffffff 128kb winargs
6fc00004-6fcffffb 64gb nsync
6fd00004-6fdffffb 64gb zipos
6fe00004-6feffffb 64gb g_fds
6ff00004-70000003 64gb free
70000000-70000001 128kb stack
70000002-700fffff 64gb free
70000004-70000004 64kb stack
70000005-70000006 128kb winargs
70000005-700fffff 64gb free
70100000-701fffff 64gb free
70200000-702fffff 64gb free
70300000-703fffff 64gb free

20
libc/runtime/stack.h
View file

@ -107,18 +107,14 @@ extern char ape_stack_align[] __attribute__((__weak__));
* problematic, since MODE=tiny doesn't use any of the runtime codes
* which want the stack to be cheaply knowable, e.g. ftrace, kprintf
*/
#define GetStaticStackAddr(ADDEND) \
({ \
intptr_t vAddr; \
if (!IsWindows() || IsAtLeastWindows10()) { \
__asm__(".weak\tape_stack_vaddr\n\t" \
"movabs\t%1+ape_stack_vaddr,%0" \
: "=r"(vAddr) \
: "i"(ADDEND)); \
} else { \
vAddr = 0x100000000 - GetStackSize(); \
} \
vAddr; \
#define GetStaticStackAddr(ADDEND) \
({ \
intptr_t vAddr; \
__asm__(".weak\tape_stack_vaddr\n\t" \
"movabs\t%1+ape_stack_vaddr,%0" \
: "=r"(vAddr) \
: "i"(ADDEND)); \
vAddr; \
})
/**

15
libc/runtime/winmain.greg.c
View file

@ -166,8 +166,8 @@ __msabi static textwindows wontreturn void WinMainNew(const char16_t *cmdline) {
char outflagsbuf[128];
const char16_t *env16;
int i, prot, count, version;
size_t allocsize, stacksize;
intptr_t stackaddr, allocaddr;
size_t allocsize, argsize, stacksize;
version = NtGetPeb()->OSMajorVersion;
__oldstack = (intptr_t)__builtin_frame_address(0);
if ((intptr_t)v_ntsubsystem == kNtImageSubsystemWindowsCui && version >= 10) {
@ -190,15 +190,14 @@ __msabi static textwindows wontreturn void WinMainNew(const char16_t *cmdline) {
rc);
}
}
_Static_assert(sizeof(struct WinArgs) % FRAMESIZE == 0, "");
_mmi.p = _mmi.s;
_mmi.n = ARRAYLEN(_mmi.s);
argsize = ROUNDUP(sizeof(struct WinArgs), FRAMESIZE);
stackaddr = GetStaticStackAddr(0);
stacksize = GetStackSize();
allocsize = argsize + stacksize;
allocaddr = stackaddr - argsize;
NTTRACE("WinMainNew() mapping %'zu byte arg block + stack at %p", allocsize,
allocaddr);
allocaddr = stackaddr;
allocsize = stacksize + sizeof(struct WinArgs);
NTTRACE("WinMainNew() mapping %'zu byte stack at %p", allocsize, allocaddr);
MapViewOfFileEx(
(_mmi.p[0].h =
CreateFileMapping(-1, &kNtIsInheritable, kNtPageExecuteReadwrite,
@ -209,12 +208,12 @@ __msabi static textwindows wontreturn void WinMainNew(const char16_t *cmdline) {
VirtualProtect((void *)allocaddr, allocsize, kNtPageReadwrite, &oldprot);
}
_mmi.p[0].x = allocaddr >> 16;
_mmi.p[0].y = (allocaddr >> 16) + ((allocsize >> 16) - 1);
_mmi.p[0].y = (allocaddr >> 16) + ((allocsize - 1) >> 16);
_mmi.p[0].prot = prot;
_mmi.p[0].flags = 0x00000026; // stack+anonymous
_mmi.p[0].size = allocsize;
_mmi.i = 1;
wa = (struct WinArgs *)allocaddr;
wa = (struct WinArgs *)(allocaddr + stacksize);
NTTRACE("WinMainNew() loading arg block");
count = GetDosArgv(cmdline, wa->argblock, ARRAYLEN(wa->argblock), wa->argv,
ARRAYLEN(wa->argv));