mirror of
https://github.com/jart/cosmopolitan.git
synced 2025-07-27 13:00:28 +00:00
Make more fixes and improvements
- Remove PAGESIZE constant
- Fix realloc() documentation
- Fix ttyname_r() error reporting
- Make forking more reliable on Windows
- Make execvp() a few microseconds faster
- Make system() a few microseconds faster
- Tighten up the socket-related magic numbers
- Loosen restrictions on mmap() offset alignment
- Improve GetProgramExecutableName() with getenv("_")
- Use mkstemp() as basis for mktemp(), tmpfile(), tmpfd()
- Fix flakes in pthread_cancel_test, unix_test, fork_test
- Fix recently introduced futex stack overflow regression
- Let sockets be passed as stdio to subprocesses on Windows
- Improve security of bind() on Windows w/ SO_EXCLUSIVEADDRUSE
This commit is contained in:
Justine Tunney
parent
140a8a52e5
commit
18bb5888e1
311 changed files with 1239 additions and 2622 deletions
356
libc/mem/arena.c
356
libc/mem/arena.c
|
|
@ -1,356 +0,0 @@
|
|||
/*-*- mode:c;indent-tabs-mode:nil;c-basic-offset:2;tab-width:8;coding:utf-8 -*-│
|
||||
│vi: set net ft=c ts=2 sts=2 sw=2 fenc=utf-8 :vi│
|
||||
╞══════════════════════════════════════════════════════════════════════════════╡
|
||||
│ Copyright 2021 Justine Alexandra Roberts Tunney │
|
||||
│ │
|
||||
│ Permission to use, copy, modify, and/or distribute this software for │
|
||||
│ any purpose with or without fee is hereby granted, provided that the │
|
||||
│ above copyright notice and this permission notice appear in all copies. │
|
||||
│ │
|
||||
│ THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL │
|
||||
│ WARRANTIES WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED │
|
||||
│ WARRANTIES OF MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE │
|
||||
│ AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL │
|
||||
│ DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR │
|
||||
│ PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER │
|
||||
│ TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR │
|
||||
│ PERFORMANCE OF THIS SOFTWARE. │
|
||||
╚─────────────────────────────────────────────────────────────────────────────*/
|
||||
#include "libc/mem/arena.h"
|
||||
#include "libc/assert.h"
|
||||
#include "libc/calls/calls.h"
|
||||
#include "libc/dce.h"
|
||||
#include "libc/intrin/bsf.h"
|
||||
#include "libc/intrin/bsr.h"
|
||||
#include "libc/intrin/likely.h"
|
||||
#include "libc/intrin/weaken.h"
|
||||
#include "libc/limits.h"
|
||||
#include "libc/log/log.h"
|
||||
#include "libc/macros.internal.h"
|
||||
#include "libc/mem/hook.internal.h"
|
||||
#include "libc/runtime/memtrack.internal.h"
|
||||
#include "libc/runtime/runtime.h"
|
||||
#include "libc/stdckdint.h"
|
||||
#include "libc/str/str.h"
|
||||
#include "libc/sysv/consts/map.h"
|
||||
#include "libc/sysv/consts/prot.h"
|
||||
#include "libc/sysv/errfuns.h"
|
||||
|
||||
#define BASE 0x50040000
|
||||
#define SIZE 0x2ff80000
|
||||
#define P(i) ((void *)(intptr_t)(i))
|
||||
#define EXCHANGE(HOOK, SLOT) \
|
||||
__arena_hook((intptr_t *)_weaken(HOOK), (intptr_t *)(&(SLOT)))
|
||||
|
||||
static struct Arena {
|
||||
bool once;
|
||||
size_t size;
|
||||
size_t depth;
|
||||
size_t offset[16];
|
||||
void (*free)(void *);
|
||||
void *(*malloc)(size_t);
|
||||
void *(*calloc)(size_t, size_t);
|
||||
void *(*memalign)(size_t, size_t);
|
||||
void *(*realloc)(void *, size_t);
|
||||
void *(*realloc_in_place)(void *, size_t);
|
||||
size_t (*malloc_usable_size)(const void *);
|
||||
size_t (*bulk_free)(void *[], size_t);
|
||||
int (*malloc_trim)(size_t);
|
||||
} __arena;
|
||||
|
||||
static wontreturn void __arena_die(void) {
|
||||
if (_weaken(__die)) _weaken(__die)();
|
||||
_exit(83);
|
||||
}
|
||||
|
||||
forceinline void __arena_check(void) {
|
||||
unassert(__arena.depth);
|
||||
}
|
||||
|
||||
forceinline void __arena_check_pointer(void *p) {
|
||||
unassert(BASE + __arena.offset[__arena.depth - 1] <= (uintptr_t)p &&
|
||||
(uintptr_t)p < BASE + __arena.offset[__arena.depth]);
|
||||
}
|
||||
|
||||
forceinline bool __arena_is_arena_pointer(void *p) {
|
||||
return BASE <= (uintptr_t)p && (uintptr_t)p < BASE + SIZE;
|
||||
}
|
||||
|
||||
forceinline size_t __arena_get_size(void *p) {
|
||||
return *(const size_t *)((const char *)p - sizeof(size_t));
|
||||
}
|
||||
|
||||
static void __arena_free(void *p) {
|
||||
__arena_check();
|
||||
if (p) {
|
||||
__arena_check_pointer(p);
|
||||
if (!(BASE <= (uintptr_t)p && (uintptr_t)p < BASE + SIZE)) {
|
||||
__arena.free(p);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
static size_t __arena_bulk_free(void *p[], size_t n) {
|
||||
size_t i;
|
||||
for (i = 0; i < n; ++i) {
|
||||
__arena_free(p[i]);
|
||||
p[i] = 0;
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
|
||||
static dontinline bool __arena_grow(size_t offset, size_t request) {
|
||||
size_t greed;
|
||||
greed = __arena.size + 1;
|
||||
do {
|
||||
greed += greed >> 1;
|
||||
greed = ROUNDUP(greed, FRAMESIZE);
|
||||
} while (greed < offset + request);
|
||||
if (greed <= SIZE) {
|
||||
if (mmap(P(BASE + __arena.size), greed - __arena.size,
|
||||
PROT_READ | PROT_WRITE, MAP_ANONYMOUS | MAP_PRIVATE | MAP_FIXED,
|
||||
-1, 0) != MAP_FAILED) {
|
||||
__arena.size = greed;
|
||||
return true;
|
||||
}
|
||||
} else {
|
||||
enomem();
|
||||
}
|
||||
if (_weaken(__oom_hook)) {
|
||||
_weaken(__oom_hook)(request);
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
static inline void *__arena_alloc(size_t a, size_t n) {
|
||||
size_t o;
|
||||
if (!n) n = 1;
|
||||
o = ROUNDUP(__arena.offset[__arena.depth] + sizeof(size_t), a);
|
||||
if (o + n >= n) {
|
||||
if (n <= sizeof(size_t)) {
|
||||
n = sizeof(size_t);
|
||||
} else {
|
||||
n = ROUNDUP(n, sizeof(size_t));
|
||||
}
|
||||
if (o + n <= SIZE) {
|
||||
if (UNLIKELY(o + n > __arena.size)) {
|
||||
if (!__arena_grow(o, n)) return 0;
|
||||
}
|
||||
__arena.offset[__arena.depth] = o + n;
|
||||
*(size_t *)(BASE + o - sizeof(size_t)) = n;
|
||||
return (void *)(BASE + o);
|
||||
}
|
||||
}
|
||||
enomem();
|
||||
return 0;
|
||||
}
|
||||
|
||||
static void *__arena_malloc(size_t n) {
|
||||
__arena_check();
|
||||
return __arena_alloc(16, n);
|
||||
}
|
||||
|
||||
static void *__arena_calloc(size_t n, size_t z) {
|
||||
__arena_check();
|
||||
if (ckd_mul(&n, n, z)) n = -1;
|
||||
return __arena_alloc(16, n);
|
||||
}
|
||||
|
||||
static void *__arena_memalign(size_t a, size_t n) {
|
||||
__arena_check();
|
||||
if (a <= sizeof(size_t)) {
|
||||
return __arena_alloc(8, n);
|
||||
} else {
|
||||
return __arena_alloc(2ul << _bsrl(a - 1), n);
|
||||
}
|
||||
}
|
||||
|
||||
static size_t __arena_malloc_usable_size(const void *p) {
|
||||
__arena_check();
|
||||
__arena_check_pointer(p);
|
||||
if (__arena_is_arena_pointer(p)) {
|
||||
return __arena_get_size(p);
|
||||
} else {
|
||||
return __arena.malloc_usable_size(p);
|
||||
}
|
||||
}
|
||||
|
||||
static void *__arena_realloc(void *p, size_t n) {
|
||||
char *q;
|
||||
size_t m, o, z;
|
||||
__arena_check();
|
||||
if (p) {
|
||||
__arena_check_pointer(p);
|
||||
if (__arena_is_arena_pointer(p)) {
|
||||
if (n) {
|
||||
if ((m = __arena_get_size(p)) >= n) {
|
||||
return p;
|
||||
} else if (n <= SIZE) {
|
||||
z = 2ul << _bsrl(n - 1);
|
||||
if (__arena.offset[__arena.depth] - m == (o = (intptr_t)p - BASE)) {
|
||||
if (UNLIKELY(o + z > __arena.size)) {
|
||||
if (o + z <= SIZE) {
|
||||
if (!__arena_grow(o, z)) {
|
||||
return 0;
|
||||
}
|
||||
} else {
|
||||
enomem();
|
||||
return 0;
|
||||
}
|
||||
}
|
||||
__arena.offset[__arena.depth] = o + z;
|
||||
*(size_t *)((char *)p - sizeof(size_t)) = z;
|
||||
return p;
|
||||
} else if ((q = __arena_alloc(1ul << _bsfl((intptr_t)p), z))) {
|
||||
memmove(q, p, m);
|
||||
return q;
|
||||
} else {
|
||||
return 0;
|
||||
}
|
||||
} else {
|
||||
enomem();
|
||||
return 0;
|
||||
}
|
||||
} else {
|
||||
return 0;
|
||||
}
|
||||
} else {
|
||||
return __arena.realloc(p, n);
|
||||
}
|
||||
} else {
|
||||
if (n <= 16) {
|
||||
n = 16;
|
||||
} else {
|
||||
n = 2ul << _bsrl(n - 1);
|
||||
}
|
||||
return __arena_alloc(16, n);
|
||||
}
|
||||
}
|
||||
|
||||
static void *__arena_realloc_in_place(void *p, size_t n) {
|
||||
char *q;
|
||||
size_t m, z;
|
||||
__arena_check();
|
||||
if (p) {
|
||||
__arena_check_pointer(p);
|
||||
if (__arena_is_arena_pointer(p)) {
|
||||
if (n) {
|
||||
if ((m = __arena_get_size(p)) >= n) {
|
||||
return p;
|
||||
} else {
|
||||
return 0;
|
||||
}
|
||||
} else {
|
||||
return 0;
|
||||
}
|
||||
} else {
|
||||
return __arena.realloc_in_place(p, n);
|
||||
}
|
||||
} else {
|
||||
return 0;
|
||||
}
|
||||
}
|
||||
|
||||
static int __arena_malloc_trim(size_t n) {
|
||||
return 0;
|
||||
}
|
||||
|
||||
static void __arena_hook(intptr_t *h, intptr_t *f) {
|
||||
intptr_t t;
|
||||
if (h) {
|
||||
t = *h;
|
||||
*h = *f;
|
||||
*f = t;
|
||||
}
|
||||
}
|
||||
|
||||
static void __arena_install(void) {
|
||||
EXCHANGE(hook_free, __arena.free);
|
||||
EXCHANGE(hook_malloc, __arena.malloc);
|
||||
EXCHANGE(hook_calloc, __arena.calloc);
|
||||
EXCHANGE(hook_realloc, __arena.realloc);
|
||||
EXCHANGE(hook_memalign, __arena.memalign);
|
||||
EXCHANGE(hook_bulk_free, __arena.bulk_free);
|
||||
EXCHANGE(hook_malloc_trim, __arena.malloc_trim);
|
||||
EXCHANGE(hook_realloc_in_place, __arena.realloc_in_place);
|
||||
EXCHANGE(hook_malloc_usable_size, __arena.malloc_usable_size);
|
||||
}
|
||||
|
||||
static void __arena_destroy(void) {
|
||||
if (__arena.depth) __arena_install();
|
||||
if (__arena.size) munmap(P(BASE), __arena.size);
|
||||
bzero(&__arena, sizeof(__arena));
|
||||
}
|
||||
|
||||
static void __arena_init(void) {
|
||||
__arena.free = __arena_free;
|
||||
__arena.malloc = __arena_malloc;
|
||||
__arena.calloc = __arena_calloc;
|
||||
__arena.realloc = __arena_realloc;
|
||||
__arena.memalign = __arena_memalign;
|
||||
__arena.bulk_free = __arena_bulk_free;
|
||||
__arena.malloc_trim = __arena_malloc_trim;
|
||||
__arena.realloc_in_place = __arena_realloc_in_place;
|
||||
__arena.malloc_usable_size = __arena_malloc_usable_size;
|
||||
atexit(__arena_destroy);
|
||||
}
|
||||
|
||||
/**
|
||||
* Pushes memory arena.
|
||||
*
|
||||
* This allocator gives a ~3x performance boost over dlmalloc, mostly
|
||||
* because it isn't thread safe and it doesn't do defragmentation.
|
||||
*
|
||||
* Calling this function will push a new arena. It may be called
|
||||
* multiple times from the main thread recursively. The first time it's
|
||||
* called, it hooks all the regular memory allocation functions. Any
|
||||
* allocations that were made previously outside the arena, will be
|
||||
* passed on to the previous hooks. Then, the basic idea, is rather than
|
||||
* bothering with free() you can just call __arena_pop() to bulk free.
|
||||
*
|
||||
* Arena allocations also have a slight size advantage, since 32-bit
|
||||
* pointers are always used. The maximum amount of arena memory is
|
||||
* 805,175,296 bytes.
|
||||
*
|
||||
* @see __arena_pop()
|
||||
*/
|
||||
void __arena_push(void) {
|
||||
if (UNLIKELY(!__arena.once)) {
|
||||
__arena_init();
|
||||
__arena.once = true;
|
||||
}
|
||||
if (!__arena.depth) {
|
||||
__arena_install();
|
||||
} else {
|
||||
unassert(__arena.depth < ARRAYLEN(__arena.offset) - 1);
|
||||
}
|
||||
__arena.offset[__arena.depth + 1] = __arena.offset[__arena.depth];
|
||||
++__arena.depth;
|
||||
}
|
||||
|
||||
/**
|
||||
* Pops memory arena.
|
||||
*
|
||||
* This pops the most recently created arena, freeing all the memory
|
||||
* that was allocated between the push and pop arena calls. If this is
|
||||
* the last arena on the stack, then the old malloc hooks are restored.
|
||||
*
|
||||
* @see __arena_push()
|
||||
*/
|
||||
void __arena_pop(void) {
|
||||
size_t a, b, greed;
|
||||
__arena_check();
|
||||
if (!--__arena.depth) __arena_install();
|
||||
a = __arena.offset[__arena.depth];
|
||||
b = __arena.offset[__arena.depth + 1];
|
||||
greed = a;
|
||||
greed += FRAMESIZE;
|
||||
greed <<= 1;
|
||||
if (__arena.size > greed) {
|
||||
munmap(P(BASE + greed), __arena.size - greed);
|
||||
__arena.size = greed;
|
||||
b = MIN(b, greed);
|
||||
a = MIN(b, a);
|
||||
}
|
||||
bzero(P(BASE + a), b - a);
|
||||
}
|
||||
|
|
@ -1,11 +0,0 @@
|
|||
#ifndef COSMOPOLITAN_LIBC_MEM_ARENA_H_
|
||||
#define COSMOPOLITAN_LIBC_MEM_ARENA_H_
|
||||
#if !(__ASSEMBLER__ + __LINKER__ + 0)
|
||||
COSMOPOLITAN_C_START_
|
||||
|
||||
void __arena_push(void);
|
||||
void __arena_pop(void);
|
||||
|
||||
COSMOPOLITAN_C_END_
|
||||
#endif /* !(__ASSEMBLER__ + __LINKER__ + 0) */
|
||||
#endif /* COSMOPOLITAN_LIBC_MEM_ARENA_H_ */
|
||||
|
|
@ -27,9 +27,7 @@ void (*hook_free)(void *) = dlfree;
|
|||
*
|
||||
* Releases the chunk of memory pointed to by p, that had been
|
||||
* previously allocated using malloc or a related routine such as
|
||||
* realloc. It has no effect if p is null. If p was not malloced or
|
||||
* already freed, free(p) will by default cause the current program to
|
||||
* abort.
|
||||
* realloc. It has no effect if p is null.
|
||||
*
|
||||
* @param p is allocation address, which may be NULL
|
||||
* @see dlfree()
|
||||
|
|
|
|||
|
|
@ -30,11 +30,14 @@ void *(*hook_malloc)(size_t) = dlmalloc;
|
|||
* on ANSI C systems.
|
||||
*
|
||||
* If n is zero, malloc returns a minimum-sized chunk. (The minimum size
|
||||
* is 32 bytes on 64bit systems.) Note that size_t is an unsigned type,
|
||||
* so calls with arguments that would be negative if signed are
|
||||
* interpreted as requests for huge amounts of space, which will often
|
||||
* fail. The maximum supported value of n differs across systems, but is
|
||||
* in all cases less than the maximum representable value of a size_t.
|
||||
* is 32 bytes on 64bit systems.) It should be assumed that zero bytes
|
||||
* are possible access, since that'll be enforced by `MODE=asan`.
|
||||
*
|
||||
* Note that size_t is an unsigned type, so calls with arguments that
|
||||
* would be negative if signed are interpreted as requests for huge
|
||||
* amounts of space, which will often fail. The maximum supported value
|
||||
* of n differs across systems, but is in all cases less than the
|
||||
* maximum representable value of a size_t.
|
||||
*
|
||||
* @param rdi is number of bytes needed, coerced to 1+
|
||||
* @return new memory, or NULL w/ errno
|
||||
|
|
|
|||
|
|
@ -16,8 +16,9 @@
|
|||
│ TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR │
|
||||
│ PERFORMANCE OF THIS SOFTWARE. │
|
||||
╚─────────────────────────────────────────────────────────────────────────────*/
|
||||
#include "libc/macros.internal.h"
|
||||
#include "libc/errno.h"
|
||||
#include "libc/mem/mem.h"
|
||||
#include "libc/stdckdint.h"
|
||||
|
||||
/**
|
||||
* Allocates granular aligned memory of granular size, i.e.
|
||||
|
|
@ -31,5 +32,9 @@
|
|||
* @threadsafe
|
||||
*/
|
||||
void *pvalloc(size_t n) {
|
||||
return memalign(FRAMESIZE, ROUNDUP(n, FRAMESIZE));
|
||||
if (ckd_add(&n, n, FRAMESIZE - 1)) {
|
||||
errno = ENOMEM;
|
||||
return 0;
|
||||
}
|
||||
return memalign(FRAMESIZE, n & -FRAMESIZE);
|
||||
}
|
||||
|
|
|
|||
|
|
@ -29,8 +29,12 @@ void *(*hook_realloc)(void *, size_t) = dlrealloc;
|
|||
* does chunk p up to the minimum of (n, p's size) bytes, or null if no
|
||||
* space is available.
|
||||
*
|
||||
* If p is NULL, realloc is equivalent to malloc.
|
||||
* If p is not NULL and n is 0, realloc is equivalent to free.
|
||||
* If p is NULL, then realloc() is equivalent to malloc().
|
||||
*
|
||||
* If p is not NULL and n is 0, then realloc() shrinks the allocation to
|
||||
* zero bytes. The allocation isn't freed and still continues to be a
|
||||
* uniquely allocated piece of memory. However it should be assumed that
|
||||
* zero bytes can be accessed, since that's enforced by `MODE=asan`.
|
||||
*
|
||||
* The returned pointer may or may not be the same as p. The algorithm
|
||||
* prefers extending p in most cases when possible, otherwise it employs
|
||||
|
|
@ -54,8 +58,6 @@ void *(*hook_realloc)(void *, size_t) = dlrealloc;
|
|||
* @param p is address of current allocation or NULL
|
||||
* @param n is number of bytes needed
|
||||
* @return rax is result, or NULL w/ errno w/o free(p)
|
||||
* @note realloc(p=0, n=0) → malloc(32)
|
||||
* @note realloc(p≠0, n=0) → free(p)
|
||||
* @see dlrealloc()
|
||||
* @threadsafe
|
||||
*/
|
||||
|
|
|
|||
|
|
@ -32,7 +32,7 @@ char *strndup(const char *s, size_t n) {
|
|||
char *s2;
|
||||
size_t len = strnlen(s, n);
|
||||
if ((s2 = malloc(len + 1))) {
|
||||
memcpy(s2, s, len);
|
||||
if (len) memcpy(s2, s, len);
|
||||
s2[len] = '\0';
|
||||
return s2;
|
||||
}
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue