mirror of
https://github.com/jart/cosmopolitan.git
synced 2025-01-31 03:27:39 +00:00
624573207e
This change doubles the performance of thread spawning. That's thanks to our new stack manager, which allows us to avoid zeroing stacks. It gives us 15µs spawns rather than 30µs spawns on Linux. Also, pthread_exit() is faster now, since it doesn't need to acquire the pthread GIL. On NetBSD, that helps us avoid allocating too many semaphores. Even if that happens we're now able to survive semaphores running out and even memory running out, when allocating *NSYNC waiter objects. I found a lot more rare bugs in the POSIX threads runtime that could cause things to crash, if you've got dozens of threads all spawning and joining dozens of threads. I want cosmo to be world class production worthy for 2025 so happy holidays all
237 lines
7.3 KiB
C++
237 lines
7.3 KiB
C++
|
|
#define LACKS_UNISTD_H
|
|
#define LACKS_FCNTL_H
|
|
#define LACKS_SYS_PARAM_H
|
|
#define LACKS_SYS_MMAN_H
|
|
#define LACKS_STRINGS_H
|
|
#define LACKS_STRING_H
|
|
#define LACKS_SYS_TYPES_H
|
|
#define LACKS_ERRNO_H
|
|
#define LACKS_STDLIB_H
|
|
#define LACKS_SCHED_H
|
|
#define LACKS_TIME_H
|
|
|
|
/* Version identifier to allow people to support multiple versions */
|
|
#ifndef DLMALLOC_VERSION
|
|
#define DLMALLOC_VERSION 20806
|
|
#endif /* DLMALLOC_VERSION */
|
|
|
|
#ifndef DLMALLOC_EXPORT
|
|
#define DLMALLOC_EXPORT extern
|
|
#endif
|
|
|
|
/* The maximum possible size_t value has all bits set */
|
|
#define MAX_SIZE_T (~(size_t)0)
|
|
|
|
#ifndef USE_LOCKS /* ensure true if spin or recursive locks set */
|
|
#define USE_LOCKS ((defined(USE_SPIN_LOCKS) && USE_SPIN_LOCKS != 0) || \
|
|
(defined(USE_RECURSIVE_LOCKS) && USE_RECURSIVE_LOCKS != 0))
|
|
#endif /* USE_LOCKS */
|
|
|
|
#if USE_LOCKS /* Spin locks for gcc >= 4.1, older gcc on x86, MSC >= 1310 */
|
|
#if ((defined(__GNUC__) && \
|
|
((__GNUC__ > 4 || (__GNUC__ == 4 && __GNUC_MINOR__ >= 1)) || \
|
|
defined(__i386__) || defined(__x86_64__))) || \
|
|
(defined(_MSC_VER) && _MSC_VER>=1310))
|
|
#ifndef USE_SPIN_LOCKS
|
|
#define USE_SPIN_LOCKS 1
|
|
#endif /* USE_SPIN_LOCKS */
|
|
#elif USE_SPIN_LOCKS
|
|
#error "USE_SPIN_LOCKS defined without implementation"
|
|
#endif /* ... locks available... */
|
|
#elif !defined(USE_SPIN_LOCKS)
|
|
#define USE_SPIN_LOCKS 0
|
|
#endif /* USE_LOCKS */
|
|
|
|
#ifndef ONLY_MSPACES
|
|
#define ONLY_MSPACES 0
|
|
#endif /* ONLY_MSPACES */
|
|
#ifndef MSPACES
|
|
#if ONLY_MSPACES
|
|
#define MSPACES 1
|
|
#else /* ONLY_MSPACES */
|
|
#define MSPACES 0
|
|
#endif /* ONLY_MSPACES */
|
|
#endif /* MSPACES */
|
|
#ifndef MALLOC_ALIGNMENT
|
|
#define MALLOC_ALIGNMENT ((size_t)(2 * sizeof(void *)))
|
|
#endif /* MALLOC_ALIGNMENT */
|
|
#ifndef FOOTERS
|
|
#define FOOTERS 0
|
|
#endif /* FOOTERS */
|
|
#ifndef ABORT
|
|
#define ABORT dlmalloc_abort()
|
|
#endif /* ABORT */
|
|
#ifndef ABORT_ON_ASSERT_FAILURE
|
|
#define ABORT_ON_ASSERT_FAILURE 1
|
|
#endif /* ABORT_ON_ASSERT_FAILURE */
|
|
#ifndef PROCEED_ON_ERROR
|
|
#define PROCEED_ON_ERROR 0
|
|
#endif /* PROCEED_ON_ERROR */
|
|
|
|
#ifndef INSECURE
|
|
#define INSECURE 0
|
|
#endif /* INSECURE */
|
|
#ifndef MALLOC_INSPECT_ALL
|
|
#define MALLOC_INSPECT_ALL 0
|
|
#endif /* MALLOC_INSPECT_ALL */
|
|
#ifndef MMAP_CLEARS
|
|
#define MMAP_CLEARS 1
|
|
#endif /* MMAP_CLEARS */
|
|
#ifndef HAVE_MREMAP
|
|
#ifdef linux
|
|
#define HAVE_MREMAP 1
|
|
#define _GNU_SOURCE /* Turns on mremap() definition */
|
|
#else /* linux */
|
|
#define HAVE_MREMAP 0
|
|
#endif /* linux */
|
|
#endif /* HAVE_MREMAP */
|
|
#ifndef MALLOC_FAILURE_ACTION
|
|
#define MALLOC_FAILURE_ACTION errno = ENOMEM;
|
|
#endif /* MALLOC_FAILURE_ACTION */
|
|
#ifndef DEFAULT_GRANULARITY
|
|
#define DEFAULT_GRANULARITY ((size_t)64U * (size_t)1024U)
|
|
#endif /* DEFAULT_GRANULARITY */
|
|
#ifndef DEFAULT_TRIM_THRESHOLD
|
|
#define DEFAULT_TRIM_THRESHOLD ((size_t)2U * (size_t)1024U * (size_t)1024U)
|
|
#endif /* DEFAULT_TRIM_THRESHOLD */
|
|
#ifndef DEFAULT_MMAP_THRESHOLD
|
|
#define DEFAULT_MMAP_THRESHOLD ((size_t)256U * (size_t)1024U)
|
|
#endif /* DEFAULT_MMAP_THRESHOLD */
|
|
#ifndef MAX_RELEASE_CHECK_RATE
|
|
#define MAX_RELEASE_CHECK_RATE 4095
|
|
#endif /* MAX_RELEASE_CHECK_RATE */
|
|
#ifndef USE_BUILTIN_FFS
|
|
#define USE_BUILTIN_FFS 0
|
|
#endif /* USE_BUILTIN_FFS */
|
|
#ifndef USE_DEV_RANDOM
|
|
#define USE_DEV_RANDOM 0
|
|
#endif /* USE_DEV_RANDOM */
|
|
#ifndef NO_MALLINFO
|
|
#define NO_MALLINFO 0
|
|
#endif /* NO_MALLINFO */
|
|
#ifndef MALLINFO_FIELD_TYPE
|
|
#define MALLINFO_FIELD_TYPE size_t
|
|
#endif /* MALLINFO_FIELD_TYPE */
|
|
#ifndef NO_MALLOC_STATS
|
|
#define NO_MALLOC_STATS 0
|
|
#endif /* NO_MALLOC_STATS */
|
|
#ifndef NO_SEGMENT_TRAVERSAL
|
|
#define NO_SEGMENT_TRAVERSAL 0
|
|
#endif /* NO_SEGMENT_TRAVERSAL */
|
|
|
|
/*
|
|
mallopt tuning options. SVID/XPG defines four standard parameter
|
|
numbers for mallopt, normally defined in malloc.h. None of these
|
|
are used in this malloc, so setting them has no effect. But this
|
|
malloc does support the following options.
|
|
*/
|
|
|
|
#define M_TRIM_THRESHOLD (-1)
|
|
#define M_GRANULARITY (-2)
|
|
#define M_MMAP_THRESHOLD (-3)
|
|
|
|
/* ------------------------ Mallinfo declarations ------------------------ */
|
|
|
|
/*
|
|
Try to persuade compilers to inline. The most critical functions for
|
|
inlining are defined as macros, so these aren't used for them.
|
|
*/
|
|
|
|
#define FORCEINLINE forceinline
|
|
#define NOINLINE dontinline
|
|
|
|
/*
|
|
========================================================================
|
|
To make a fully customizable malloc.h header file, cut everything
|
|
#include "libc/sysv/consts/map.h"
|
|
#include "libc/runtime/runtime.h"
|
|
above this line, put into file malloc.h, edit to suit, and #include it
|
|
on the next line, as well as in programs that use this malloc.
|
|
========================================================================
|
|
*/
|
|
|
|
/* ------------------- size_t and alignment properties -------------------- */
|
|
|
|
/* The byte and bit size of a size_t */
|
|
#define SIZE_T_SIZE (sizeof(size_t))
|
|
#define SIZE_T_BITSIZE (sizeof(size_t) << 3)
|
|
|
|
/* Some constants coerced to size_t */
|
|
/* Annoying but necessary to avoid errors on some platforms */
|
|
#define SIZE_T_ZERO ((size_t)0)
|
|
#define SIZE_T_ONE ((size_t)1)
|
|
#define SIZE_T_TWO ((size_t)2)
|
|
#define SIZE_T_FOUR ((size_t)4)
|
|
#define TWO_SIZE_T_SIZES (SIZE_T_SIZE<<1)
|
|
#define FOUR_SIZE_T_SIZES (SIZE_T_SIZE<<2)
|
|
#define SIX_SIZE_T_SIZES (FOUR_SIZE_T_SIZES+TWO_SIZE_T_SIZES)
|
|
#define HALF_MAX_SIZE_T (MAX_SIZE_T / 2U)
|
|
|
|
/* The bit mask value corresponding to MALLOC_ALIGNMENT */
|
|
#define CHUNK_ALIGN_MASK (MALLOC_ALIGNMENT - SIZE_T_ONE)
|
|
|
|
/* True if address a has acceptable alignment */
|
|
#define is_aligned(A) (((size_t)((A)) & (CHUNK_ALIGN_MASK)) == 0)
|
|
|
|
/* the number of bytes to offset an address to align it */
|
|
#define align_offset(A)\
|
|
((((size_t)(A) & CHUNK_ALIGN_MASK) == 0)? 0 :\
|
|
((MALLOC_ALIGNMENT - ((size_t)(A) & CHUNK_ALIGN_MASK)) & CHUNK_ALIGN_MASK))
|
|
|
|
/* -------------------------- MMAP preliminaries ------------------------- */
|
|
|
|
/* MORECORE and MMAP must return MFAIL on failure */
|
|
#define MFAIL NULL
|
|
#define CMFAIL ((char*)(MFAIL)) /* defined for convenience */
|
|
|
|
#define MUNMAP_DEFAULT(a, s) munmap((a), (s))
|
|
#define MMAP_PROT (PROT_READ|PROT_WRITE)
|
|
#define MMAP_FLAGS (MAP_PRIVATE|MAP_ANONYMOUS)
|
|
#define MMAP_DEFAULT(s) _mapanon(s)
|
|
#define DIRECT_MMAP_DEFAULT(s) MMAP_DEFAULT(s)
|
|
|
|
#if HAVE_MREMAP
|
|
#ifndef WIN32
|
|
#define MREMAP_DEFAULT(addr, osz, nsz, mv) mremap((addr), (osz), (nsz), (mv))
|
|
#endif /* WIN32 */
|
|
#endif /* HAVE_MREMAP */
|
|
|
|
/**
|
|
* Define CALL_MMAP/CALL_MUNMAP/CALL_DIRECT_MMAP
|
|
*/
|
|
#define USE_MMAP_BIT (SIZE_T_ONE)
|
|
|
|
#ifdef MMAP
|
|
#define CALL_MMAP(s) MMAP(s)
|
|
#else /* MMAP */
|
|
#define CALL_MMAP(s) MMAP_DEFAULT(s)
|
|
#endif /* MMAP */
|
|
|
|
#ifdef MUNMAP
|
|
#define CALL_MUNMAP(a, s) MUNMAP((a), (s))
|
|
#else /* MUNMAP */
|
|
#define CALL_MUNMAP(a, s) MUNMAP_DEFAULT((a), (s))
|
|
#endif /* MUNMAP */
|
|
|
|
#ifdef DIRECT_MMAP
|
|
#define CALL_DIRECT_MMAP(s) DIRECT_MMAP(s)
|
|
#else /* DIRECT_MMAP */
|
|
#define CALL_DIRECT_MMAP(s) DIRECT_MMAP_DEFAULT(s)
|
|
#endif /* DIRECT_MMAP */
|
|
|
|
/**
|
|
* Define CALL_MREMAP
|
|
*/
|
|
#if HAVE_MREMAP
|
|
#define CALL_MREMAP(addr, osz, nsz, mv) ({ int olderr = errno; void *res = mremap((addr), (osz), (nsz), (mv)); if (res == MAP_FAILED) errno = olderr; res; })
|
|
#else /* HAVE_MREMAP */
|
|
#define CALL_MREMAP(addr, osz, nsz, mv) MAP_FAILED
|
|
#endif /* HAVE_MREMAP */
|
|
|
|
/* mstate bit set if continguous morecore disabled or failed */
|
|
#define USE_NONCONTIGUOUS_BIT (4U)
|
|
|
|
/* segment bit set in create_mspace_with_base */
|
|
#define EXTERN_BIT (8U)
|
|
|