mirrors/cosmopolitan
1
0
Fork 0
mirror of https://github.com/jart/cosmopolitan.git synced 2025-07-27 04:50:28 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

Make fixes, improvements, and chibicc python bindings

Browse source 
- python now mixes audio 10x faster
- python octal notation is restored
- chibicc now builds code 3x faster
- chibicc now has help documentation
- chibicc can now generate basic python bindings
- linenoise now supports some paredit-like features

See #141
This commit is contained in:
Justine Tunney 2021-10-08 08:11:51 -07:00
parent 28997f3acb
commit 7061c79c22
121 changed files with 5272 additions and 1928 deletions
Download patch file Download diff file Expand all files Collapse all files

3
libc/alg/replacestr.c
View file

@ -19,6 +19,7 @@
#include "libc/alg/alg.h"
#include "libc/alg/arraylist2.internal.h"
#include "libc/bits/safemacros.internal.h"
#include "libc/macros.internal.h"
#include "libc/str/str.h"
#include "libc/sysv/errfuns.h"
@ -38,7 +39,7 @@ char *replacestr(const char *s, const char *needle, const char *replacement) {
nlen = strlen(needle);
rlen = strlen(replacement);
res_i = 0;
res_n = max(left, 32);
res_n = MAX(left, 32);
if ((res_p = malloc(res_n * sizeof(char)))) {
do {
if (!(p2 = memmem(p1, left, needle, nlen))) break;

2
libc/bits/roundup2pow.c
View file

@ -26,5 +26,5 @@
* @see rounddown2pow()
*/
unsigned long roundup2pow(unsigned long x) {
return x > 1 ? 1ul << (bsrl(x - 1) + 1) : x ? 1 : 0;
return x > 1 ? 2ul << bsrl(x - 1) : x ? 1 : 0;
}

55
libc/fmt/formatint32.c Normal file
View file

@ -0,0 +1,55 @@
/*-*- mode:c;indent-tabs-mode:nil;c-basic-offset:2;tab-width:8;coding:utf-8 -*-│
│vi: set net ft=c ts=2 sts=2 sw=2 fenc=utf-8 :vi│
╞══════════════════════════════════════════════════════════════════════════════╡
│ Copyright 2021 Justine Alexandra Roberts Tunney │
│ │
│ Permission to use, copy, modify, and/or distribute this software for │
│ any purpose with or without fee is hereby granted, provided that the │
│ above copyright notice and this permission notice appear in all copies. │
│ │
│ THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL │
│ WARRANTIES WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED │
│ WARRANTIES OF MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE │
│ AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL │
│ DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR │
│ PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER │
│ TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR │
│ PERFORMANCE OF THIS SOFTWARE. │
╚─────────────────────────────────────────────────────────────────────────────*/
#include "libc/fmt/itoa.h"
/**
* Converts unsigned 32-bit integer to string.
*
* @param p needs at least 12 bytes
* @return pointer to nul byte
*/
noinline char *FormatUint32(char p[static 12], uint32_t x) {
char t;
size_t i, a, b;
i = 0;
do {
p[i++] = x % 10 + '0';
x = x / 10;
} while (x > 0);
p[i] = '\0';
if (i) {
for (a = 0, b = i - 1; a < b; ++a, --b) {
t = p[a];
p[a] = p[b];
p[b] = t;
}
}
return p + i;
}
/**
* Converts signed 32-bit integer to string.
*
* @param p needs at least 12 bytes
* @return pointer to nul byte
*/
char *FormatInt32(char p[static 12], int32_t x) {
if (x < 0) *p++ = '-', x = -(uint32_t)x;
return FormatUint32(p, x);
}

2
libc/fmt/itoa.h
View file

@ -16,6 +16,8 @@ COSMOPOLITAN_C_START_
- uint128toarray_radix10(0x31337, a) l: 93 (27ns) m: 141 (41ns)
- int128toarray_radix10(0x31337, a) l: 96 (28ns) m: 173 (51ns) */
char *FormatInt32(char[hasatleast 12], int32_t);
char *FormatUint32(char[hasatleast 12], uint32_t);
char *FormatInt64(char[hasatleast 21], int64_t);
char *FormatUint64(char[hasatleast 21], uint64_t);
char *FormatInt64Thousands(char[hasatleast 27], int64_t);

30
libc/intrin/asan.c
View file

@ -871,16 +871,16 @@ static size_t __asan_malloc_usable_size(const void *p) {
static void __asan_deallocate(char *p, long kind) {
size_t c, n;
if ((c = weaken(dlmalloc_usable_size)(p)) >= 8) {
if (__asan_read48(p + c - 8, &n) && n <= c) {
__asan_poison((uintptr_t)p, c, kind);
if (c <= FRAMESIZE) {
p = __asan_morgue_add(p);
}
weaken(dlfree)(p);
} else {
__asan_report_invalid_pointer(p);
if (__asan_is_mapped((intptr_t)p >> 16) &&
(((intptr_t)p >> 16) == ((intptr_t)(p - 16) >> 16) ||
__asan_is_mapped((intptr_t)(p - 16) >> 16)) &&
(c = weaken(dlmalloc_usable_size)(p)) >= 8 &&
__asan_read48(p + c - 8, &n) && n <= c) {
__asan_poison((uintptr_t)p, c, kind);
if (c <= FRAMESIZE) {
p = __asan_morgue_add(p);
}
weaken(dlfree)(p);
} else {
__asan_report_invalid_pointer(p);
}
@ -891,6 +891,17 @@ void __asan_free(void *p) {
__asan_deallocate(p, kAsanHeapFree);
}
size_t __asan_bulk_free(void *p[], size_t n) {
size_t i;
for (i = 0; i < n; ++i) {
if (p[i]) {
__asan_deallocate(p[i], kAsanHeapFree);
p[i] = 0;
}
}
return 0;
}
void *__asan_memalign(size_t align, size_t size) {
return __asan_allocate(align, size, kAsanHeapUnderrun, kAsanHeapOverrun);
}
@ -1058,6 +1069,7 @@ void __asan_install_malloc_hooks(void) {
HOOK(hook_pvalloc, __asan_pvalloc);
HOOK(hook_realloc, __asan_realloc);
HOOK(hook_memalign, __asan_memalign);
HOOK(hook_bulk_free, __asan_bulk_free);
HOOK(hook_malloc_trim, __asan_malloc_trim);
HOOK(hook_realloc_in_place, __asan_realloc_in_place);
HOOK(hook_malloc_usable_size, __asan_malloc_usable_size);

10
libc/intrin/bzero.c
View file

@ -18,6 +18,7 @@
╚─────────────────────────────────────────────────────────────────────────────*/
#include "libc/assert.h"
#include "libc/dce.h"
#include "libc/intrin/asan.internal.h"
#include "libc/nexgen32e/nexgen32e.h"
#include "libc/nexgen32e/x86feature.h"
#include "libc/str/str.h"
@ -25,8 +26,9 @@
typedef char xmm_t __attribute__((__vector_size__(16), __aligned__(1)));
typedef long long xmm_a __attribute__((__vector_size__(16), __aligned__(16)));
static noinline antiquity void bzero_sse(char *p, size_t n) {
noasan static noinline antiquity void bzero_sse(char *p, size_t n) {
xmm_t v = {0};
if (IsAsan()) __asan_check(p, n);
if (n <= 32) {
*(xmm_t *)(p + n - 16) = v;
*(xmm_t *)p = v;
@ -41,12 +43,13 @@ static noinline antiquity void bzero_sse(char *p, size_t n) {
}
}
microarchitecture("avx") static void bzero_avx(char *p, size_t n) {
noasan microarchitecture("avx") static void bzero_avx(char *p, size_t n) {
xmm_t v = {0};
if (IsAsan()) __asan_check(p, n);
if (n <= 32) {
*(xmm_t *)(p + n - 16) = v;
*(xmm_t *)p = v;
} else if (!IsAsan() && n >= 1024 && X86_HAVE(ERMS)) {
} else if (n >= 1024 && X86_HAVE(ERMS)) {
asm("rep stosb" : "+D"(p), "+c"(n), "=m"(*(char(*)[n])p) : "a"(0));
} else {
if (n < kHalfCache3 || !kHalfCache3) {
@ -132,6 +135,7 @@ void(bzero)(void *p, size_t n) {
uint64_t x;
b = p;
if (IsTiny()) {
if (IsAsan()) __asan_check(p, n);
asm("rep stosb" : "+D"(b), "+c"(n), "=m"(*(char(*)[n])b) : "0"(p), "a"(0));
return;
}

41
libc/intrin/memcmp.c
View file

@ -29,8 +29,8 @@ static noinline antiquity int memcmp_sse(const unsigned char *p,
unsigned u, u0, u1, u2, u3;
if (n > 32) {
while (n > 16 + 16) {
if (!(u = __builtin_ia32_pmovmskb128(__builtin_ia32_pcmpeqb128(
*(const xmm_t *)p, *(const xmm_t *)q)) -
if (!(u = __builtin_ia32_pmovmskb128(*(const xmm_t *)p ==
*(const xmm_t *)q) -
0xffff)) {
n -= 16;
p += 16;
@ -41,11 +41,10 @@ static noinline antiquity int memcmp_sse(const unsigned char *p,
}
}
}
if (!(u = __builtin_ia32_pmovmskb128(__builtin_ia32_pcmpeqb128(
*(const xmm_t *)p, *(const xmm_t *)q)) -
if (!(u = __builtin_ia32_pmovmskb128(*(const xmm_t *)p == *(const xmm_t *)q) -
0xffff)) {
if (!(u = __builtin_ia32_pmovmskb128(__builtin_ia32_pcmpeqb128(
*(const xmm_t *)(p + n - 16), *(const xmm_t *)(q + n - 16))) -
if (!(u = __builtin_ia32_pmovmskb128(*(const xmm_t *)(p + n - 16) ==
*(const xmm_t *)(q + n - 16)) -
0xffff)) {
return 0;
} else {
@ -65,14 +64,14 @@ microarchitecture("avx") static int memcmp_avx(const unsigned char *p,
unsigned u, u0, u1, u2, u3;
if (n > 32) {
while (n >= 16 + 64) {
u0 = __builtin_ia32_pmovmskb128(__builtin_ia32_pcmpeqb128(
((const xmm_t *)p)[0], ((const xmm_t *)q)[0]));
u1 = __builtin_ia32_pmovmskb128(__builtin_ia32_pcmpeqb128(
((const xmm_t *)p)[1], ((const xmm_t *)q)[1]));
u2 = __builtin_ia32_pmovmskb128(__builtin_ia32_pcmpeqb128(
((const xmm_t *)p)[2], ((const xmm_t *)q)[2]));
u3 = __builtin_ia32_pmovmskb128(__builtin_ia32_pcmpeqb128(
((const xmm_t *)p)[3], ((const xmm_t *)q)[3]));
u0 = __builtin_ia32_pmovmskb128(
(((const xmm_t *)p)[0] == ((const xmm_t *)q)[0]));
u1 = __builtin_ia32_pmovmskb128(
(((const xmm_t *)p)[1] == ((const xmm_t *)q)[1]));
u2 = __builtin_ia32_pmovmskb128(
(((const xmm_t *)p)[2] == ((const xmm_t *)q)[2]));
u3 = __builtin_ia32_pmovmskb128(
(((const xmm_t *)p)[3] == ((const xmm_t *)q)[3]));
w = (uint64_t)u0 | (uint64_t)u1 << 16 | (uint64_t)u2 << 32 |
(uint64_t)u3 << 48;
if (w == -1) {
@ -85,8 +84,8 @@ microarchitecture("avx") static int memcmp_avx(const unsigned char *p,
}
}
while (n > 16 + 16) {
if (!(u = __builtin_ia32_pmovmskb128(__builtin_ia32_pcmpeqb128(
*(const xmm_t *)p, *(const xmm_t *)q)) -
if (!(u = __builtin_ia32_pmovmskb128(*(const xmm_t *)p ==
*(const xmm_t *)q) -
0xffff)) {
n -= 16;
p += 16;
@ -97,11 +96,10 @@ microarchitecture("avx") static int memcmp_avx(const unsigned char *p,
}
}
}
if (!(u = __builtin_ia32_pmovmskb128(__builtin_ia32_pcmpeqb128(
*(const xmm_t *)p, *(const xmm_t *)q)) -
if (!(u = __builtin_ia32_pmovmskb128(*(const xmm_t *)p == *(const xmm_t *)q) -
0xffff)) {
if (!(u = __builtin_ia32_pmovmskb128(__builtin_ia32_pcmpeqb128(
*(const xmm_t *)(p + n - 16), *(const xmm_t *)(q + n - 16))) -
if (!(u = __builtin_ia32_pmovmskb128(*(const xmm_t *)(p + n - 16) ==
*(const xmm_t *)(q + n - 16)) -
0xffff)) {
return 0;
} else {
@ -149,7 +147,8 @@ int memcmp(const void *a, const void *b, size_t n) {
uint32_t k, i, j;
uint64_t w, x, y;
const unsigned char *p, *q;
if ((p = a) == (q = b)) return 0;
if ((p = a) == (q = b) || !n) return 0;
if ((c = *p - *q)) return c;
if (!IsTiny()) {
if (n <= 16) {
if (n >= 8) {

92
libc/intrin/memmove.c
View file

@ -89,8 +89,8 @@ asm("memcpy = memmove\n\t"
* @asyncsignalsafe
*/
void *memmove(void *dst, const void *src, size_t n) {
char *d;
size_t i;
char *d, *r;
const char *s;
uint64_t a, b;
xmm_t v, w, x, y, V, W, X, Y, wut;
@ -119,18 +119,22 @@ void *memmove(void *dst, const void *src, size_t n) {
} else if (n) {
*d = *s;
}
} else if (d <= s) {
asm("rep movsb"
: "+D"(d), "+S"(s), "+c"(n), "=m"(*(char(*)[n])dst)
: "m"(*(char(*)[n])src));
} else {
d += n - 1;
s += n - 1;
asm("std\n\t"
"rep movsb\n\t"
"cld"
: "+D"(d), "+S"(s), "+c"(n), "=m"(*(char(*)[n])dst)
: "m"(*(char(*)[n])src));
if (IsAsan()) __asan_check(d, n);
if (IsAsan()) __asan_check(s, n);
if (d <= s) {
asm("rep movsb"
: "+D"(d), "+S"(s), "+c"(n), "=m"(*(char(*)[n])dst)
: "m"(*(char(*)[n])src));
} else {
d += n - 1;
s += n - 1;
asm("std\n\t"
"rep movsb\n\t"
"cld"
: "+D"(d), "+S"(s), "+c"(n), "=m"(*(char(*)[n])dst)
: "m"(*(char(*)[n])src));
}
}
return dst;
}
@ -208,7 +212,6 @@ void *memmove(void *dst, const void *src, size_t n) {
*(xmm_t *)(d + n - 16) = Y;
return d;
default:
r = d;
if (d == s) return d;
if (n < kHalfCache3 || !kHalfCache3) {
if (d > s) {
@ -221,12 +224,14 @@ void *memmove(void *dst, const void *src, size_t n) {
*(xmm_t *)(d + n + 16) = w;
} while (n >= 32);
} else {
if (IsAsan()) __asan_check(d, n);
if (IsAsan()) __asan_check(s, n);
asm("std\n\t"
"rep movsb\n\t"
"cld"
: "=D"(d), "=S"(s), "+c"(n), "=m"(*(char(*)[n])d)
: "0"(d + n - 1), "1"(s + n - 1), "m"(*(char(*)[n])s));
return r;
return dst;
}
} else {
if (IsAsan() || n < 900 || !X86_HAVE(ERMS)) {
@ -241,10 +246,12 @@ void *memmove(void *dst, const void *src, size_t n) {
s += i;
n -= i;
} else {
if (IsAsan()) __asan_check(d, n);
if (IsAsan()) __asan_check(s, n);
asm("rep movsb"
: "+D"(d), "+S"(s), "+c"(n), "=m"(*(char(*)[n])d)
: "m"(*(char(*)[n])s));
return r;
return dst;
}
}
} else {
@ -278,54 +285,31 @@ void *memmove(void *dst, const void *src, size_t n) {
}
asm("sfence");
}
switch (n) {
case 0:
return r;
case 17 ... 31:
__builtin_memcpy(&v, s, 16);
__builtin_memcpy(&w, s + n - 16, 16);
__builtin_memcpy(d, &v, 16);
__builtin_memcpy(d + n - 16, &w, 16);
return r;
case 9 ... 15:
if (n) {
if (n >= 16) {
v = *(const xmm_t *)s;
w = *(const xmm_t *)(s + n - 16);
*(xmm_t *)d = v;
*(xmm_t *)(d + n - 16) = w;
} else if (n >= 8) {
__builtin_memcpy(&a, s, 8);
__builtin_memcpy(&b, s + n - 8, 8);
__builtin_memcpy(d, &a, 8);
__builtin_memcpy(d + n - 8, &b, 8);
return r;
case 5 ... 7:
} else if (n >= 4) {
__builtin_memcpy(&a, s, 4);
__builtin_memcpy(&b, s + n - 4, 4);
__builtin_memcpy(d, &a, 4);
__builtin_memcpy(d + n - 4, &b, 4);
return r;
case 16:
__builtin_memcpy(&v, s, 16);
__builtin_memcpy(d, &v, 16);
return r;
case 8:
__builtin_memcpy(&a, s, 8);
__builtin_memcpy(d, &a, 8);
return r;
case 4:
__builtin_memcpy(&a, s, 4);
__builtin_memcpy(d, &a, 4);
return r;
case 1:
} else if (n >= 2) {
__builtin_memcpy(&a, s, 2);
__builtin_memcpy(&b, s + n - 2, 2);
__builtin_memcpy(d, &a, 2);
__builtin_memcpy(d + n - 2, &b, 2);
} else {
*d = *s;
return r;
case 2:
__builtin_memcpy(&a, s, 2);
__builtin_memcpy(d, &a, 2);
return r;
case 3:
__builtin_memcpy(&a, s, 2);
__builtin_memcpy(&b, s + 1, 2);
__builtin_memcpy(d, &a, 2);
__builtin_memcpy(d + 1, &b, 2);
return r;
default:
unreachable;
}
}
return dst;
}
}

11
libc/intrin/memset.c
View file

@ -18,6 +18,7 @@
╚─────────────────────────────────────────────────────────────────────────────*/
#include "libc/assert.h"
#include "libc/dce.h"
#include "libc/intrin/asan.internal.h"
#include "libc/nexgen32e/nexgen32e.h"
#include "libc/nexgen32e/x86feature.h"
#include "libc/str/str.h"
@ -25,8 +26,9 @@
typedef char xmm_t __attribute__((__vector_size__(16), __aligned__(1)));
typedef long long xmm_a __attribute__((__vector_size__(16), __aligned__(16)));
static noinline antiquity void *memset_sse(char *p, char c, size_t n) {
noasan static noinline antiquity void *memset_sse(char *p, char c, size_t n) {
xmm_t v = {c, c, c, c, c, c, c, c, c, c, c, c, c, c, c, c};
if (IsAsan()) __asan_check(p, n);
if (n <= 32) {
*(xmm_t *)(p + n - 16) = v;
*(xmm_t *)p = v;
@ -42,13 +44,15 @@ static noinline antiquity void *memset_sse(char *p, char c, size_t n) {
return p;
}
microarchitecture("avx") static void *memset_avx(char *p, char c, size_t n) {
noasan microarchitecture("avx") static void *memset_avx(char *p, char c,
size_t n) {
char *t;
xmm_t v = {c, c, c, c, c, c, c, c, c, c, c, c, c, c, c, c};
if (IsAsan()) __asan_check(p, n);
if (n <= 32) {
*(xmm_t *)(p + n - 16) = v;
*(xmm_t *)p = v;
} else if (!IsAsan() && n >= 1024 && X86_HAVE(ERMS)) {
} else if (n >= 1024 && X86_HAVE(ERMS)) {
asm("rep stosb" : "=D"(t), "+c"(n), "=m"(*(char(*)[n])p) : "0"(p), "a"(c));
} else {
if (n < kHalfCache3 || !kHalfCache3) {
@ -137,6 +141,7 @@ void *memset(void *p, int c, size_t n) {
uint64_t x;
b = p;
if (IsTiny()) {
if (IsAsan()) __asan_check(p, n);
asm("rep stosb" : "+D"(b), "+c"(n), "=m"(*(char(*)[n])b) : "0"(p), "a"(c));
return p;
}

4
libc/intrin/strlen.c
View file

@ -35,8 +35,8 @@ noasan size_t strlen(const char *s) {
unsigned m, k = (uintptr_t)s & 15;
const xmm_t *p = (const xmm_t *)((uintptr_t)s & -16);
if (IsAsan()) __asan_verify(s, 1);
m = __builtin_ia32_pmovmskb128(__builtin_ia32_pcmpeqb128(*p, z)) >> k << k;
while (!m) m = __builtin_ia32_pmovmskb128(__builtin_ia32_pcmpeqb128(*++p, z));
m = __builtin_ia32_pmovmskb128(*p == z) >> k << k;
while (!m) m = __builtin_ia32_pmovmskb128(*++p == z);
n = (const char *)p + __builtin_ctzl(m) - s;
if (IsAsan()) __asan_verify(s, n);
return n;

2
libc/log/backtrace2.c
View file

@ -111,6 +111,7 @@ static noasan int PrintBacktraceUsingAddr2line(int fd,
*
* Then it's unpleasant to need to press C-x C-n six times.
*/
#if 0
while ((p2 = memchr(p1, '\n', p3 - p1))) {
if (memmem(p1, p2 - p1, ": __asan_", 9) ||
memmem(p1, p2 - p1, ": __die", 7)) {
@ -121,6 +122,7 @@ static noasan int PrintBacktraceUsingAddr2line(int fd,
break;
}
}
#endif
/*
* remove racist output from gnu tooling, that can't be disabled

1
libc/macros.internal.h
View file

@ -15,6 +15,7 @@
#define alignas(x) _Alignas(x)
#define IS2POW(X) (!((X) & ((X)-1)))
#define ROUNDUP(X, K) (((X) + (K)-1) & -(K))
#define ROUNDDOWN(X, K) ((X) & -(K))
#define ABS(X) ((X) >= 0 ? (X) : -(X))

238
libc/mem/arena.c Normal file
View file

@ -0,0 +1,238 @@
/*-*- mode:c;indent-tabs-mode:nil;c-basic-offset:2;tab-width:8;coding:utf-8 -*-│
│vi: set net ft=c ts=2 sts=2 sw=2 fenc=utf-8 :vi│
╞══════════════════════════════════════════════════════════════════════════════╡
│ Copyright 2021 Justine Alexandra Roberts Tunney │
│ │
│ Permission to use, copy, modify, and/or distribute this software for │
│ any purpose with or without fee is hereby granted, provided that the │
│ above copyright notice and this permission notice appear in all copies. │
│ │
│ THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL │
│ WARRANTIES WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED │
│ WARRANTIES OF MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE │
│ AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL │
│ DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR │
│ PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER │
│ TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR │
│ PERFORMANCE OF THIS SOFTWARE. │
╚─────────────────────────────────────────────────────────────────────────────*/
#include "libc/assert.h"
#include "libc/bits/likely.h"
#include "libc/bits/weaken.h"
#include "libc/calls/calls.h"
#include "libc/dce.h"
#include "libc/limits.h"
#include "libc/log/libfatal.internal.h"
#include "libc/log/log.h"
#include "libc/macros.internal.h"
#include "libc/mem/arena.h"
#include "libc/mem/hook/hook.internal.h"
#include "libc/runtime/runtime.h"
#include "libc/sysv/consts/map.h"
#include "libc/sysv/consts/prot.h"
#define BASE ((char *)0x30000000)
#define LIMIT ((char *)0x50000000)
#define EXCHANGE(HOOK, SLOT) \
__arena_hook((intptr_t *)weaken(HOOK), (intptr_t *)(&(SLOT)))
static struct Arena {
bool once;
uint8_t depth;
unsigned size;
unsigned offset[16];
void (*free)(void *);
void *(*malloc)(size_t);
void *(*calloc)(size_t, size_t);
void *(*memalign)(size_t, size_t);
void *(*realloc)(void *, size_t);
void *(*realloc_in_place)(void *, size_t);
void *(*valloc)(size_t);
void *(*pvalloc)(size_t);
int (*malloc_trim)(size_t);
size_t (*malloc_usable_size)(const void *);
size_t (*bulk_free)(void *[], size_t);
} __arena;
static wontreturn void __arena_die(void) {
if (weaken(__die)) weaken(__die)();
_exit(83);
}
static wontreturn void __arena_not_implemented(void) {
__printf("not implemented");
__arena_die();
}
static void __arena_free(void *p) {
if (!p) return;
assert(__arena.depth);
assert((intptr_t)BASE + __arena.offset[__arena.depth - 1] <= (intptr_t)p &&
(intptr_t)p < (intptr_t)BASE + __arena.offset[__arena.depth]);
}
static size_t __arena_bulk_free(void *p[], size_t n) {
size_t i;
for (i = 0; i < n; ++i) {
if (p[i]) __arena_free(p[i]);
}
bzero(p, n * sizeof(void *));
return 0;
}
static void *__arena_malloc(size_t n) {
char *ptr;
size_t need, greed;
assert(__arena.depth);
if (!n) n = 1;
if (n < LIMIT - BASE) {
need = __arena.offset[__arena.depth] + n;
need = ROUNDUP(need, __BIGGEST_ALIGNMENT__);
if (UNLIKELY(need > __arena.size)) {
greed = __arena.size + 1;
do {
greed += greed >> 1;
greed = ROUNDUP(greed, FRAMESIZE);
} while (need > greed);
if (greed < LIMIT - BASE &&
mmap(BASE + __arena.size, greed - __arena.size,
PROT_READ | PROT_WRITE, MAP_ANONYMOUS | MAP_PRIVATE | MAP_FIXED,
-1, 0) != MAP_FAILED) {
__arena.size = greed;
} else {
return 0;
}
}
ptr = BASE + __arena.offset[__arena.depth];
__arena.offset[__arena.depth] = need;
return ptr;
} else {
return 0;
}
}
static void *__arena_calloc(size_t n, size_t z) {
if (__builtin_mul_overflow(n, z, &n)) n = -1;
return __arena_malloc(n);
}
static void *__arena_memalign(size_t a, size_t n) {
if (a <= __BIGGEST_ALIGNMENT__) {
return __arena_malloc(n);
} else {
__arena_not_implemented();
}
}
static void *__arena_realloc(void *p, size_t n) {
if (p) {
if (n) {
__arena_not_implemented();
} else {
__arena_free(p);
return 0;
}
} else {
return __arena_malloc(n);
}
}
static int __arena_malloc_trim(size_t n) {
return 0;
}
static void *__arena_realloc_in_place(void *p, size_t n) {
__arena_not_implemented();
}
static void *__arena_valloc(size_t n) {
__arena_not_implemented();
}
static void *__arena_pvalloc(size_t n) {
__arena_not_implemented();
}
static size_t __arena_malloc_usable_size(const void *p) {
__arena_not_implemented();
}
static void __arena_hook(intptr_t *h, intptr_t *f) {
intptr_t t;
if (h) {
t = *h;
*h = *f;
*f = t;
}
}
static void __arena_install(void) {
EXCHANGE(hook_free, __arena.free);
EXCHANGE(hook_realloc, __arena.realloc);
EXCHANGE(hook_realloc, __arena.realloc);
EXCHANGE(hook_malloc, __arena.malloc);
EXCHANGE(hook_calloc, __arena.calloc);
EXCHANGE(hook_memalign, __arena.memalign);
EXCHANGE(hook_realloc_in_place, __arena.realloc_in_place);
EXCHANGE(hook_valloc, __arena.valloc);
EXCHANGE(hook_pvalloc, __arena.pvalloc);
EXCHANGE(hook_malloc_trim, __arena.malloc_trim);
EXCHANGE(hook_malloc_usable_size, __arena.malloc_usable_size);
EXCHANGE(hook_bulk_free, __arena.bulk_free);
}
static void __arena_destroy(void) {
if (__arena.depth) {
__arena_install();
}
if (__arena.size) {
munmap(BASE, __arena.size);
}
bzero(&__arena, sizeof(__arena));
}
static void __arena_init(void) {
__arena.free = __arena_free;
__arena.realloc = __arena_realloc;
__arena.realloc = __arena_realloc;
__arena.malloc = __arena_malloc;
__arena.calloc = __arena_calloc;
__arena.memalign = __arena_memalign;
__arena.realloc_in_place = __arena_realloc_in_place;
__arena.valloc = __arena_valloc;
__arena.pvalloc = __arena_pvalloc;
__arena.malloc_trim = __arena_malloc_trim;
__arena.malloc_usable_size = __arena_malloc_usable_size;
__arena.bulk_free = __arena_bulk_free;
atexit(__arena_destroy);
}
void __arena_push(void) {
if (UNLIKELY(!__arena.once)) {
__arena_init();
__arena.once = true;
}
if (!__arena.depth) {
__arena_install();
} else if (__arena.depth == ARRAYLEN(__arena.offset) - 1) {
__printf("too many arenas");
__arena_die();
}
__arena.offset[__arena.depth + 1] = __arena.offset[__arena.depth];
++__arena.depth;
}
void __arena_pop(void) {
unsigned greed;
assert(__arena.depth);
bzero(BASE + __arena.offset[__arena.depth - 1],
__arena.offset[__arena.depth] - __arena.offset[__arena.depth - 1]);
if (!--__arena.depth) __arena_install();
greed = __arena.offset[__arena.depth];
greed += FRAMESIZE;
greed <<= 1;
if (__arena.size > greed) {
munmap(BASE + greed, __arena.size - greed);
}
}

11
libc/mem/arena.h Normal file
View file

@ -0,0 +1,11 @@
#ifndef COSMOPOLITAN_LIBC_MEM_ARENA_H_
#define COSMOPOLITAN_LIBC_MEM_ARENA_H_
#if !(__ASSEMBLER__ + __LINKER__ + 0)
COSMOPOLITAN_C_START_
void __arena_push(void);
void __arena_pop(void);
COSMOPOLITAN_C_END_
#endif /* !(__ASSEMBLER__ + __LINKER__ + 0) */
#endif /* COSMOPOLITAN_LIBC_MEM_ARENA_H_ */

30
libc/mem/bulk_free.S Normal file
View file

@ -0,0 +1,30 @@
/*-*- mode:unix-assembly; indent-tabs-mode:t; tab-width:8; coding:utf-8 -*-│
│vi: set et ft=asm ts=8 tw=8 fenc=utf-8 :vi│
╞══════════════════════════════════════════════════════════════════════════════╡
│ Copyright 2020 Justine Alexandra Roberts Tunney │
│ │
│ Permission to use, copy, modify, and/or distribute this software for │
│ any purpose with or without fee is hereby granted, provided that the │
│ above copyright notice and this permission notice appear in all copies. │
│ │
│ THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL │
│ WARRANTIES WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED │
│ WARRANTIES OF MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE │
│ AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL │
│ DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR │
│ PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER │
│ TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR │
│ PERFORMANCE OF THIS SOFTWARE. │
╚─────────────────────────────────────────────────────────────────────────────*/
#include "libc/macros.internal.h"
// Frees and clears (sets to NULL) each non-null pointer in given array.
//
// This is twice as fast as freeing them one-by-one. If footers are
// used, pointers that have been allocated in different mspaces are
// not freed or cleared, and the count of all such pointers is returned.
// For large arrays of pointers with poor locality, it may be worthwhile
// to sort this array before calling bulk_free.
bulk_free:
jmp *hook_bulk_free(%rip)
.endfn bulk_free,globl

31
libc/mem/hook/bulk_free.S Normal file
View file

@ -0,0 +1,31 @@
/*-*- mode:unix-assembly; indent-tabs-mode:t; tab-width:8; coding:utf-8 -*-│
│vi: set et ft=asm ts=8 tw=8 fenc=utf-8 :vi│
╞══════════════════════════════════════════════════════════════════════════════╡
│ Copyright 2020 Justine Alexandra Roberts Tunney │
│ │
│ Permission to use, copy, modify, and/or distribute this software for │
│ any purpose with or without fee is hereby granted, provided that the │
│ above copyright notice and this permission notice appear in all copies. │
│ │
│ THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL │
│ WARRANTIES WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED │
│ WARRANTIES OF MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE │
│ AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL │
│ DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR │
│ PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER │
│ TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR │
│ PERFORMANCE OF THIS SOFTWARE. │
╚─────────────────────────────────────────────────────────────────────────────*/
#include "libc/macros.internal.h"
.initbss 202,_init_bulk_free
hook_bulk_free:
.quad 0
.endobj hook_bulk_free,globl,hidden
.previous
.init.start 202,_init_bulk_free
.hidden dlbulk_free
ezlea dlbulk_free,ax
stosq
.init.end 202,_init_bulk_free

1
libc/mem/hook/hook.internal.h
View file

@ -13,6 +13,7 @@ extern void *(*hook_valloc)(size_t);
extern void *(*hook_pvalloc)(size_t);
extern int (*hook_malloc_trim)(size_t);
extern size_t (*hook_malloc_usable_size)(const void *);
extern size_t (*hook_bulk_free)(void *[], size_t);
COSMOPOLITAN_C_END_
#endif /* !(__ASSEMBLER__ + __LINKER__ + 0) */

7
libc/nt/efi.h
View file

@ -101,7 +101,12 @@
#if !(__ASSEMBLER__ + __LINKER__ + 0)
COSMOPOLITAN_C_START_
#define EFIAPI __attribute__((__ms_abi__))
#if defined(__GNUC__) && __GNUC__ >= 6 && !defined(__chibicc__)
#define EFIAPI __attribute__((__ms_abi__))
#else
#define EFIAPI /* TODO(jart): fix me */
#endif
#define EFI_STATUS uint64_t
#define EFI_EVENT uintptr_t
#define EFI_HANDLE uintptr_t

6
libc/rand/rngset.c
View file

@ -17,6 +17,7 @@
│ PERFORMANCE OF THIS SOFTWARE. │
╚─────────────────────────────────────────────────────────────────────────────*/
#include "libc/calls/calls.h"
#include "libc/dce.h"
#include "libc/intrin/asan.internal.h"
#include "libc/rand/rand.h"
#include "libc/stdio/stdio.h"
@ -40,10 +41,13 @@
*
* @return original buf
*/
void *rngset(void *b, size_t n, uint64_t seed(void), size_t reseed) {
noasan void *rngset(void *b, size_t n, uint64_t seed(void), size_t reseed) {
size_t m;
uint64_t i, x, t = 0;
unsigned char *p = b;
if (IsAsan()) {
__asan_check(b, n);
}
if (!seed) {
t = reseed;
reseed = -1;

2
libc/runtime/atexit.c
View file

@ -29,5 +29,5 @@
* @return 0 on success or nonzero if out of space
*/
int atexit(void f(void)) {
return __cxa_atexit(f, NULL, NULL);
return __cxa_atexit(f, 0, 0);
}

2
libc/runtime/cxaatexit.c
View file

@ -51,7 +51,7 @@ static struct CxaAtexitBlocks {
* @return 0 on success or nonzero w/ errno
* @note folks have forked libc in past just to unbloat atexit()
*/
int __cxa_atexit(void *fp, void *arg, void *pred) {
noasan int __cxa_atexit(void *fp, void *arg, void *pred) {
unsigned i;
struct CxaAtexitBlock *b, *b2;
_Static_assert(ATEXIT_MAX == CHAR_BIT * sizeof(b->mask), "");

16
libc/runtime/hook.greg.c
View file

@ -21,9 +21,11 @@
#include "libc/calls/internal.h"
#include "libc/calls/sigbits.h"
#include "libc/calls/struct/sigset.h"
#include "libc/errno.h"
#include "libc/log/libfatal.internal.h"
#include "libc/runtime/runtime.h"
#include "libc/runtime/symbols.internal.h"
#include "libc/str/str.h"
#include "libc/sysv/consts/prot.h"
/**
@ -44,8 +46,9 @@
*
* @see ape/ape.lds
*/
privileged noinstrument noasan void __hook(void *ifunc,
struct SymbolTable *symbols) {
privileged noinstrument noasan int __hook(void *ifunc,
struct SymbolTable *symbols) {
int rc;
size_t i;
char *p, *pe;
intptr_t addr;
@ -57,10 +60,10 @@ privileged noinstrument noasan void __hook(void *ifunc,
bool kIsBinaryAligned = !(kPrivilegedStart & (PAGESIZE - 1));
sigfillset(&mask);
sigprocmask(SIG_BLOCK, &mask, &oldmask);
if (mprotect((void *)symbols->addr_base,
kPrivilegedStart - symbols->addr_base,
kIsBinaryAligned ? PROT_READ | PROT_WRITE
: PROT_READ | PROT_WRITE | PROT_EXEC) != -1) {
if ((rc = mprotect(
(void *)symbols->addr_base, kPrivilegedStart - symbols->addr_base,
kIsBinaryAligned ? PROT_READ | PROT_WRITE
: PROT_READ | PROT_WRITE | PROT_EXEC)) != -1) {
for (i = 0; i < symbols->count; ++i) {
if (symbols->addr_base + symbols->symbols[i].x < kProgramCodeStart) {
continue;
@ -125,4 +128,5 @@ privileged noinstrument noasan void __hook(void *ifunc,
PROT_READ | PROT_EXEC);
}
sigprocmask(SIG_SETMASK, &oldmask, NULL);
return rc;
}

2
libc/runtime/memtrack.internal.h
View file

@ -19,7 +19,7 @@ COSMOPOLITAN_C_START_
(!(IsWindows() && NtGetVersion() < kNtVersionWindows10) ? NORMAL : WIN7)
#define kAutomapStart MEMTRACK_ADDRESS(_kAutomapStart, 0x10000000)
#define kAutomapSize MEMTRACK_ADDRESS(_kAutomapSize, 0x40000000)
#define kFixedmapStart MEMTRACK_ADDRESS(_kFixedmapStart, 0x40000000)
#define kFixedmapStart MEMTRACK_ADDRESS(_kFixedmapStart, 0x50000000)
struct MemoryInterval {
int x;

2
libc/runtime/symbols.internal.h
View file

@ -26,7 +26,7 @@ const char *FindComBinary(void);
const char *FindDebugBinary(void);
struct SymbolTable *OpenSymbolTable(const char *);
int CloseSymbolTable(struct SymbolTable **);
void __hook(void *, struct SymbolTable *);
int __hook(void *, struct SymbolTable *);
forceinline int GetSymbol(struct SymbolTable *t, intptr_t a) {
unsigned l, m, r, n, k;

45
libc/str/djbsort.c
View file

@ -17,18 +17,55 @@
│ PERFORMANCE OF THIS SOFTWARE. │
╚─────────────────────────────────────────────────────────────────────────────*/
#include "libc/alg/alg.h"
#include "libc/dce.h"
#include "libc/intrin/asan.internal.h"
#include "libc/nexgen32e/bsr.h"
#include "libc/nexgen32e/nexgen32e.h"
#include "libc/nexgen32e/x86feature.h"
void djbsort_avx2(int32_t *, long);
static noinline void intsort(int *x, size_t n, size_t t) {
int a, b, c;
size_t i, p, q;
for (p = t; p > 0; p >>= 1) {
for (i = 0; i < n - p; ++i) {
if (!(i & p)) {
a = x[i + 0];
b = x[i + p];
if (a > b) c = a, a = b, b = c;
x[i + 0] = a;
x[i + p] = b;
}
}
for (q = t; q > p; q >>= 1) {
for (i = 0; i < n - q; ++i) {
if (!(i & p)) {
a = x[i + p];
b = x[i + q];
if (a > b) c = a, a = b, b = c;
x[i + p] = a;
x[i + q] = b;
}
}
}
}
}
/**
* D.J. Bernstein's outrageously fast integer sorting algorithm.
*/
void djbsort(int32_t *a, size_t n) {
if (X86_HAVE(AVX2)) {
djbsort_avx2(a, n);
} else {
insertionsort(a, n);
size_t m;
if (IsAsan()) {
if (__builtin_mul_overflow(n, 4, &m)) m = -1;
__asan_check(a, m);
}
if (n > 1) {
if (X86_HAVE(AVX2)) {
djbsort_avx2(a, n);
} else {
intsort(a, n, 1ul << bsrl(n - 1));
}
}
}

10
libc/str/iswseparator.c
View file

@ -390,14 +390,14 @@ static const unsigned kAstralCodes[][2] = {
* other things like blocks and emoji (So).
*/
int iswseparator(wint_t c) {
int m, l, r;
int m, l, r, n;
if (c < 0200) {
return !(('0' <= c && c <= '9') || ('A' <= c && c <= 'Z') ||
('a' <= c && c <= 'z'));
}
if (c <= 0xffff) {
l = 0;
r = sizeof(kCodes) / sizeof(kCodes[0]);
r = n = sizeof(kCodes) / sizeof(kCodes[0]);
while (l < r) {
m = (l + r) >> 1;
if (kCodes[m][1] < c) {
@ -406,10 +406,10 @@ int iswseparator(wint_t c) {
r = m;
}
}
return !(kCodes[l][0] <= c && c <= kCodes[l][1]);
return !(l < n && kCodes[l][0] <= c && c <= kCodes[l][1]);
} else {
l = 0;
r = sizeof(kAstralCodes) / sizeof(kAstralCodes[0]);
r = n = sizeof(kAstralCodes) / sizeof(kAstralCodes[0]);
while (l < r) {
m = (l + r) >> 1;
if (kAstralCodes[m][1] < c) {
@ -418,6 +418,6 @@ int iswseparator(wint_t c) {
r = m;
}
}
return !(kAstralCodes[l][0] <= c && c <= kAstralCodes[l][1]);
return !(l < n && kAstralCodes[l][0] <= c && c <= kAstralCodes[l][1]);
}
}

38
libc/str/khextoint.c Normal file
View file

@ -0,0 +1,38 @@
/*-*- mode:c;indent-tabs-mode:nil;c-basic-offset:2;tab-width:8;coding:utf-8 -*-│
│vi: set net ft=c ts=2 sts=2 sw=2 fenc=utf-8 :vi│
╞══════════════════════════════════════════════════════════════════════════════╡
│ Copyright 2021 Justine Alexandra Roberts Tunney │
│ │
│ Permission to use, copy, modify, and/or distribute this software for │
│ any purpose with or without fee is hereby granted, provided that the │
│ above copyright notice and this permission notice appear in all copies. │
│ │
│ THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL │
│ WARRANTIES WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED │
│ WARRANTIES OF MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE │
│ AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL │
│ DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR │
│ PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER │
│ TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR │
│ PERFORMANCE OF THIS SOFTWARE. │
╚─────────────────────────────────────────────────────────────────────────────*/
#include "libc/str/str.h"
const signed char kHexToInt[256] = {
-1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, // 0x00
-1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, // 0x10
-1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, // 0x20
0, 1, 2, 3, 4, 5, 6, 7, 8, 9, -1, -1, -1, -1, -1, -1, // 0x30
-1, 10, 11, 12, 13, 14, 15, -1, -1, -1, -1, -1, -1, -1, -1, -1, // 0x40
-1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, // 0x50
-1, 10, 11, 12, 13, 14, 15, -1, -1, -1, -1, -1, -1, -1, -1, -1, // 0x60
-1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, // 0x70
-1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, // 0x80
-1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, // 0x90
-1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, // 0xa0
-1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, // 0xb0
-1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, // 0xc0
-1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, // 0xd0
-1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, // 0xe0
-1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, // 0xf0
};

11
libc/runtime/longsort.c → libc/str/longsort.c
View file

@ -18,12 +18,14 @@
╚─────────────────────────────────────────────────────────────────────────────*/
#include "libc/dce.h"
#include "libc/intrin/asan.internal.h"
#include "libc/macros.internal.h"
#include "libc/nexgen32e/bsr.h"
#include "libc/nexgen32e/x86feature.h"
#include "libc/runtime/runtime.h"
forceinline void longsorter(long *x, size_t n, size_t t) {
long a, b, c, p, q, i;
long a, b, c;
size_t i, p, q;
for (p = t; p > 0; p >>= 1) {
for (i = 0; i < n - p; ++i) {
if (!(i & p)) {
@ -68,7 +70,10 @@ void longsort(long *x, size_t n) {
}
if (n > 1) {
t = 1ul << bsrl(n - 1);
if (X86_HAVE(AVX2)) return longsort_avx2(x, n, t);
return longsort_pure(x, n, t);
if (X86_HAVE(AVX2)) {
longsort_avx2(x, n, t);
} else {
longsort_pure(x, n, t);
}
}
}

34
libc/str/memccpy.c
View file

@ -54,38 +54,12 @@ static inline noasan uint64_t UncheckedAlignedRead64(unsigned char *p) {
*/
void *memccpy(void *dst, const void *src, int c, size_t n) {
size_t i;
uint64_t v, w;
unsigned char *d, *q;
unsigned char *d;
const unsigned char *s;
i = 0;
d = dst;
s = src;
c &= 255;
v = 0x0101010101010101ul * c;
for (; (uintptr_t)(s + i) & 7; ++i) {
if (i == n) return NULL;
if ((d[i] = s[i]) == c) return d + i + 1;
}
for (; i + 8 <= n; i += 8) {
w = UncheckedAlignedRead64(s + i);
if (~(w ^ v) & ((w ^ v) - 0x0101010101010101) & 0x8080808080808080) {
break;
} else {
q = d + i;
q[0] = (w & 0x00000000000000ff) >> 000;
q[1] = (w & 0x000000000000ff00) >> 010;
q[2] = (w & 0x0000000000ff0000) >> 020;
q[3] = (w & 0x00000000ff000000) >> 030;
q[4] = (w & 0x000000ff00000000) >> 040;
q[5] = (w & 0x0000ff0000000000) >> 050;
q[6] = (w & 0x00ff000000000000) >> 060;
q[7] = (w & 0xff00000000000000) >> 070;
}
}
for (; i < n; ++i) {
if ((d[i] = s[i]) == c) {
for (d = dst, s = src, i = 0; i < n; ++i) {
if ((d[i] = s[i]) == (c & 255)) {
return d + i + 1;
}
}
return NULL;
return 0;
}

69
libc/str/memchr.c
View file

@ -16,35 +16,64 @@
│ TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR │
│ PERFORMANCE OF THIS SOFTWARE. │
╚─────────────────────────────────────────────────────────────────────────────*/
#include "libc/dce.h"
#include "libc/intrin/asan.internal.h"
#include "libc/nexgen32e/x86feature.h"
#include "libc/str/str.h"
typedef char xmm_t __attribute__((__vector_size__(16), __aligned__(1)));
static inline const unsigned char *memchr_pure(const unsigned char *s,
unsigned char c, size_t n) {
size_t i;
for (i = 0; i < n; ++i) {
if (s[i] == c) {
return s + i;
}
}
return 0;
}
noasan static inline const unsigned char *memchr_sse(const unsigned char *s,
unsigned char c,
size_t n) {
size_t i;
unsigned k;
unsigned m;
xmm_t v, *p;
xmm_t t = {c, c, c, c, c, c, c, c, c, c, c, c, c, c, c, c};
for (; n >= 16; n -= 16, s += 16) {
v = *(const xmm_t *)s;
m = __builtin_ia32_pmovmskb128(v == t);
if (m) {
m = __builtin_ctzll(m);
return s + m;
}
}
for (i = 0; i < n; ++i) {
if (s[i] == c) {
return s + i;
}
}
return 0;
}
/**
* Returns pointer to first instance of character.
*
* @param m is memory to search
* @param s is memory to search
* @param c is search byte which is masked with 255
* @param n is byte length of p
* @return is pointer to first instance of c or NULL if not found
* @asyncsignalsafe
*/
void *memchr(const void *m, int c, size_t n) {
uint64_t v, w;
const char *p, *pe;
c &= 255;
v = 0x0101010101010101ul * c;
for (p = m, pe = p + n; p + 8 <= pe; p += 8) {
w = (uint64_t)(255 & p[7]) << 070 | (uint64_t)(255 & p[6]) << 060 |
(uint64_t)(255 & p[5]) << 050 | (uint64_t)(255 & p[4]) << 040 |
(uint64_t)(255 & p[3]) << 030 | (uint64_t)(255 & p[2]) << 020 |
(uint64_t)(255 & p[1]) << 010 | (uint64_t)(255 & p[0]) << 000;
if ((w = ~(w ^ v) & ((w ^ v) - 0x0101010101010101) & 0x8080808080808080)) {
return p + ((unsigned)__builtin_ctzll(w) >> 3);
}
void *memchr(const void *s, int c, size_t n) {
const void *r;
if (X86_HAVE(SSE)) {
if (IsAsan()) __asan_check(s, n);
r = memchr_sse(s, c, n);
} else {
r = memchr_pure(s, c, n);
}
for (; p < pe; ++p) {
if ((*p & 255) == c) {
return p;
}
}
return NULL;
return (void *)r;
}

0
libc/runtime/qsort.c → libc/str/qsort.c
View file

60
libc/str/rawmemchr.c
View file

@ -17,12 +17,39 @@
│ PERFORMANCE OF THIS SOFTWARE. │
╚─────────────────────────────────────────────────────────────────────────────*/
#include "libc/assert.h"
#include "libc/dce.h"
#include "libc/intrin/asan.internal.h"
#include "libc/nexgen32e/x86feature.h"
#include "libc/str/str.h"
static inline noasan uint64_t UncheckedAlignedRead64(unsigned char *p) {
return (uint64_t)p[7] << 070 | (uint64_t)p[6] << 060 | (uint64_t)p[5] << 050 |
(uint64_t)p[4] << 040 | (uint64_t)p[3] << 030 | (uint64_t)p[2] << 020 |
(uint64_t)p[1] << 010 | (uint64_t)p[0] << 000;
typedef char xmm_t __attribute__((__vector_size__(16), __aligned__(16)));
static inline const unsigned char *rawmemchr_pure(const unsigned char *s,
unsigned char c) {
for (;; ++s) {
if (*s == c) {
return s;
}
}
}
noasan static inline const char *rawmemchr_sse(const char *s, unsigned char c) {
unsigned k;
unsigned m;
xmm_t v, *p;
xmm_t n = {c, c, c, c, c, c, c, c, c, c, c, c, c, c, c, c};
k = (uintptr_t)s & 15;
p = (const xmm_t *)((uintptr_t)s & -16);
v = *p;
m = __builtin_ia32_pmovmskb128(v == n);
m >>= k;
m <<= k;
while (!m) {
v = *++p;
m = __builtin_ia32_pmovmskb128(v == n);
}
m = __builtin_ctzll(m);
return (const char *)p + m;
}
/**
@ -32,22 +59,13 @@ static inline noasan uint64_t UncheckedAlignedRead64(unsigned char *p) {
* @param c is search byte which is masked with 255
* @return is pointer to first instance of c
*/
void *rawmemchr(const void *m, int c) {
uint64_t v, w;
const unsigned char *s;
s = m;
c &= 255;
v = 0x0101010101010101ul * c;
for (; (uintptr_t)s & 7; ++s) {
if (*s == c) return s;
void *rawmemchr(const void *s, int c) {
const void *r;
if (X86_HAVE(SSE)) {
if (IsAsan()) __asan_check(s, 1);
r = rawmemchr_sse(s, c);
} else {
r = rawmemchr_pure(s, c);
}
for (;; s += 8) {
w = UncheckedAlignedRead64(s);
if ((w = ~(w ^ v) & ((w ^ v) - 0x0101010101010101) & 0x8080808080808080)) {
s += (unsigned)__builtin_ctzll(w) >> 3;
break;
}
}
assert(*s == c);
return s;
return (void *)r;
}

1
libc/str/str.h
View file

@ -7,6 +7,7 @@ COSMOPOLITAN_C_START_
╚────────────────────────────────────────────────────────────────────────────│─╝
fourth age telecommunications */
extern const int8_t kHexToInt[256];
extern const uint8_t gperf_downcase[256];
extern const uint8_t kToLower[256];
extern const uint8_t kToUpper[256];

72
libc/str/strchr.c
View file

@ -17,37 +17,42 @@
│ PERFORMANCE OF THIS SOFTWARE. │
╚─────────────────────────────────────────────────────────────────────────────*/
#include "libc/assert.h"
#include "libc/dce.h"
#include "libc/intrin/asan.internal.h"
#include "libc/nexgen32e/x86feature.h"
#include "libc/str/str.h"
static noasan inline const char *strchr_x64(const char *p, uint64_t c) {
unsigned a, b;
uint64_t w, x, y;
for (c *= 0x0101010101010101;; p += 8) {
w = (uint64_t)(255 & p[7]) << 070 | (uint64_t)(255 & p[6]) << 060 |
(uint64_t)(255 & p[5]) << 050 | (uint64_t)(255 & p[4]) << 040 |
(uint64_t)(255 & p[3]) << 030 | (uint64_t)(255 & p[2]) << 020 |
(uint64_t)(255 & p[1]) << 010 | (uint64_t)(255 & p[0]) << 000;
if ((x = ~(w ^ c) & ((w ^ c) - 0x0101010101010101) & 0x8080808080808080) |
(y = ~w & (w - 0x0101010101010101) & 0x8080808080808080)) {
if (x) {
a = __builtin_ctzll(x);
if (y) {
b = __builtin_ctzll(y);
if (a <= b) {
return p + (a >> 3);
} else {
return 0;
}
} else {
return p + (a >> 3);
}
} else {
return 0;
}
}
typedef char xmm_t __attribute__((__vector_size__(16), __aligned__(16)));
static inline const char *strchr_pure(const char *s, int c) {
for (;; ++s) {
if ((*s & 255) == (c & 255)) return s;
if (!*s) return 0;
}
}
noasan static inline const char *strchr_sse(const char *s, unsigned char c) {
unsigned k;
unsigned m;
xmm_t v, *p;
xmm_t z = {0};
xmm_t n = {c, c, c, c, c, c, c, c, c, c, c, c, c, c, c, c};
k = (uintptr_t)s & 15;
p = (const xmm_t *)((uintptr_t)s & -16);
v = *p;
m = __builtin_ia32_pmovmskb128((v == z) | (v == n));
m >>= k;
m <<= k;
while (!m) {
v = *++p;
m = __builtin_ia32_pmovmskb128((v == z) | (v == n));
}
m = __builtin_ctzl(m);
s = (const char *)p + m;
if (c && !*s) s = 0;
return s;
}
/**
* Returns pointer to first instance of character.
*
@ -58,12 +63,13 @@ static noasan inline const char *strchr_x64(const char *p, uint64_t c) {
* @asyncsignalsafe
*/
char *strchr(const char *s, int c) {
char *r;
for (c &= 255; (uintptr_t)s & 7; ++s) {
if ((*s & 255) == c) return s;
if (!*s) return NULL;
const char *r;
if (X86_HAVE(SSE)) {
if (IsAsan()) __asan_check(s, 1);
r = strchr_sse(s, c);
} else {
r = strchr_pure(s, c);
}
r = strchr_x64(s, c);
assert(!r || *r || !c);
return r;
assert(!r || *r || !(c & 255));
return (char *)r;
}

70
libc/str/strchrnul.c
View file

@ -17,38 +17,39 @@
│ PERFORMANCE OF THIS SOFTWARE. │
╚─────────────────────────────────────────────────────────────────────────────*/
#include "libc/assert.h"
#include "libc/dce.h"
#include "libc/intrin/asan.internal.h"
#include "libc/nexgen32e/x86feature.h"
#include "libc/str/str.h"
noasan static const char *strchrnul_x64(const char *p, uint64_t c) {
unsigned a, b;
uint64_t w, x, y;
for (c *= 0x0101010101010101;; p += 8) {
w = (uint64_t)(255 & p[7]) << 070 | (uint64_t)(255 & p[6]) << 060 |
(uint64_t)(255 & p[5]) << 050 | (uint64_t)(255 & p[4]) << 040 |
(uint64_t)(255 & p[3]) << 030 | (uint64_t)(255 & p[2]) << 020 |
(uint64_t)(255 & p[1]) << 010 | (uint64_t)(255 & p[0]) << 000;
if ((x = ~(w ^ c) & ((w ^ c) - 0x0101010101010101) & 0x8080808080808080) |
(y = ~w & (w - 0x0101010101010101) & 0x8080808080808080)) {
if (x) {
a = __builtin_ctzll(x);
if (y) {
b = __builtin_ctzll(y);
if (a <= b) {
return p + (a >> 3);
} else {
return p + (b >> 3);
}
} else {
return p + (a >> 3);
}
} else {
b = __builtin_ctzll(y);
return p + (b >> 3);
}
}
typedef char xmm_t __attribute__((__vector_size__(16), __aligned__(16)));
static inline const char *strchrnul_pure(const char *s, int c) {
for (;; ++s) {
if ((*s & 255) == (c & 255)) return s;
if (!*s) return s;
}
}
noasan static inline const char *strchrnul_sse(const char *s, unsigned char c) {
unsigned k;
unsigned m;
xmm_t v, *p;
xmm_t z = {0};
xmm_t n = {c, c, c, c, c, c, c, c, c, c, c, c, c, c, c, c};
k = (uintptr_t)s & 15;
p = (const xmm_t *)((uintptr_t)s & -16);
v = *p;
m = __builtin_ia32_pmovmskb128((v == z) | (v == n));
m >>= k;
m <<= k;
while (!m) {
v = *++p;
m = __builtin_ia32_pmovmskb128((v == z) | (v == n));
}
return (const char *)p + __builtin_ctzl(m);
}
/**
* Returns pointer to first instance of character.
*
@ -58,12 +59,13 @@ noasan static const char *strchrnul_x64(const char *p, uint64_t c) {
* NUL terminator if c is not found
*/
char *strchrnul(const char *s, int c) {
char *r;
for (c &= 255; (uintptr_t)s & 7; ++s) {
if ((*s & 0xff) == c) return s;
if (!*s) return s;
const char *r;
if (X86_HAVE(SSE)) {
if (IsAsan()) __asan_check(s, 1);
r = strchrnul_sse(s, c);
} else {
r = strchrnul_pure(s, c);
}
r = strchrnul_x64(s, c);
assert((*r & 255) == c || !*r);
return r;
assert((*r & 255) == (c & 255) || !*r);
return (char *)r;
}

2
libc/str/strcmp.c
View file

@ -34,9 +34,11 @@ static inline noasan uint64_t UncheckedAlignedRead64(const char *p) {
* @asyncsignalsafe
*/
int strcmp(const char *a, const char *b) {
int c;
size_t i = 0;
uint64_t v, w, d;
if (a == b) return 0;
if ((c = (*a & 255) - (*b & 255))) return c;
if (((uintptr_t)a & 7) == ((uintptr_t)b & 7)) {
for (; (uintptr_t)(a + i) & 7; ++i) {
if (a[i] != b[i] || !b[i]) {

15
libc/str/strpbrk.c
View file

@ -16,7 +16,6 @@
│ TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR │
│ PERFORMANCE OF THIS SOFTWARE. │
╚─────────────────────────────────────────────────────────────────────────────*/
#include "libc/nexgen32e/hascharacter.internal.h"
#include "libc/str/str.h"
/**
@ -24,17 +23,21 @@
* @asyncsignalsafe
*/
char *strpbrk(const char *s, const char *accept) {
size_t i;
bool lut[256];
if (accept[0]) {
if (!accept[1]) {
return strchr(s, accept[0]);
} else {
for (i = 0; s[i]; ++i) {
if (HasCharacter(s[i], accept)) {
return (/*unconst*/ char *)&s[i];
memset(lut, 0, sizeof(lut));
while (*accept) {
lut[*accept++ & 255] = true;
}
for (; *s; ++s) {
if (lut[*s & 255]) {
return (/*unconst*/ char *)s;
}
}
}
}
return NULL;
return 0;
}

6
libc/str/strstr.c
View file

@ -49,14 +49,12 @@ noasan char *strstr(const char *haystack, const char *needle) {
k = (uintptr_t)haystack & 15;
p = (const xmm_t *)((uintptr_t)haystack & -16);
v = *p;
m = __builtin_ia32_pmovmskb128(__builtin_ia32_pcmpeqb128(v, z) |
__builtin_ia32_pcmpeqb128(v, n));
m = __builtin_ia32_pmovmskb128((v == z) | (v == n));
m >>= k;
m <<= k;
while (!m) {
v = *++p;
m = __builtin_ia32_pmovmskb128(__builtin_ia32_pcmpeqb128(v, z) |
__builtin_ia32_pcmpeqb128(v, n));
m = __builtin_ia32_pmovmskb128((v == z) | (v == n));
}
haystack = (const char *)p + __builtin_ctzl(m);
for (i = 0;; ++i) {

25
libc/str/timingsafe_bcmp.c
View file

@ -18,14 +18,15 @@
╚─────────────────────────────────────────────────────────────────────────────*/
#include "libc/bits/likely.h"
#include "libc/dce.h"
#include "libc/intrin/asan.internal.h"
#include "libc/nexgen32e/x86feature.h"
#include "libc/str/str.h"
typedef uint64_t xmm_t __attribute__((__vector_size__(16), __aligned__(1)));
static noinline antiquity unsigned timingsafe_bcmp_sse(const char *p,
const char *q,
size_t n) {
noasan static noinline antiquity unsigned timingsafe_bcmp_sse(const char *p,
const char *q,
size_t n) {
uint64_t w;
xmm_t a = {0};
while (n > 16 + 16) {
@ -40,9 +41,9 @@ static noinline antiquity unsigned timingsafe_bcmp_sse(const char *p,
return w | w >> 32;
}
microarchitecture("avx") static int timingsafe_bcmp_avx(const char *p,
const char *q,
size_t n) {
noasan static microarchitecture("avx") int timingsafe_bcmp_avx(const char *p,
const char *q,
size_t n) {
uint64_t w;
xmm_t a = {0};
if (n > 32) {
@ -134,10 +135,16 @@ int timingsafe_bcmp(const void *a, const void *b, size_t n) {
__builtin_memcpy(&w3, q + n - 8, 8);
w = (w0 ^ w1) | (w2 ^ w3);
return w | w >> 32;
} else if (X86_HAVE(AVX)) {
return timingsafe_bcmp_avx(p, q, n);
} else {
return timingsafe_bcmp_sse(p, q, n);
if (IsAsan()) {
__asan_check(a, n);
__asan_check(b, n);
}
if (X86_HAVE(AVX)) {
return timingsafe_bcmp_avx(p, q, n);
} else {
return timingsafe_bcmp_sse(p, q, n);
}
}
} else if (n >= 4) {
__builtin_memcpy(&u0, p, 4);

10
libc/str/towlower.c
View file

@ -177,7 +177,7 @@ static const int kAstralLower[][3] = {
* Converts wide character to lower case.
*/
wint_t towlower(wint_t c) {
int m, l, r;
int m, l, r, n;
if (c < 0200) {
if ('A' <= c && c <= 'Z') {
return c + 32;
@ -199,7 +199,7 @@ wint_t towlower(wint_t c) {
return c + 38864; /* 80x Ꭰ ..Ꮿ → ꭰ ..ꮿ Cherokee */
} else {
l = 0;
r = sizeof(kLower) / sizeof(kLower[0]);
r = n = sizeof(kLower) / sizeof(kLower[0]);
while (l < r) {
m = (l + r) >> 1;
if (kLower[m].y < c) {
@ -208,7 +208,7 @@ wint_t towlower(wint_t c) {
r = m;
}
}
if (kLower[l].x <= c && c <= kLower[l].y) {
if (l < n && kLower[l].x <= c && c <= kLower[l].y) {
return c + kLower[l].d;
} else {
return c;
@ -216,7 +216,7 @@ wint_t towlower(wint_t c) {
}
} else {
l = 0;
r = sizeof(kAstralLower) / sizeof(kAstralLower[0]);
r = n = sizeof(kAstralLower) / sizeof(kAstralLower[0]);
while (l < r) {
m = (l + r) >> 1;
if (kAstralLower[m][1] < c) {
@ -225,7 +225,7 @@ wint_t towlower(wint_t c) {
r = m;
}
}
if (kAstralLower[l][0] <= c && c <= kAstralLower[l][1]) {
if (l < n && kAstralLower[l][0] <= c && c <= kAstralLower[l][1]) {
return c + kAstralLower[l][2];
} else {
return c;

10
libc/str/towupper.c
View file

@ -140,7 +140,7 @@ static const int kAstralUpper[][3] = {
* Converts wide character to upper case.
*/
wint_t towupper(wint_t c) {
int m, l, r;
int m, l, r, n;
if (c < 0200) {
if ('a' <= c && c <= 'z') {
return c - 32;
@ -162,7 +162,7 @@ wint_t towupper(wint_t c) {
return c - 38864; /* 80x ꭰ ..ꮿ → Ꭰ ..Ꮿ Cherokee Supplement */
} else {
l = 0;
r = sizeof(kUpper) / sizeof(kUpper[0]);
r = n = sizeof(kUpper) / sizeof(kUpper[0]);
while (l < r) {
m = (l + r) >> 1;
if (kUpper[m].y < c) {
@ -171,7 +171,7 @@ wint_t towupper(wint_t c) {
r = m;
}
}
if (kUpper[l].x <= c && c <= kUpper[l].y) {
if (l < n && kUpper[l].x <= c && c <= kUpper[l].y) {
return c + kUpper[l].d;
} else {
return c;
@ -179,7 +179,7 @@ wint_t towupper(wint_t c) {
}
} else {
l = 0;
r = sizeof(kAstralUpper) / sizeof(kAstralUpper[0]);
r = n = sizeof(kAstralUpper) / sizeof(kAstralUpper[0]);
while (l < r) {
m = (l + r) >> 1;
if (kAstralUpper[m][1] < c) {
@ -188,7 +188,7 @@ wint_t towupper(wint_t c) {
r = m;
}
}
if (kAstralUpper[l][0] <= c && c <= kAstralUpper[l][1]) {
if (l < n && kAstralUpper[l][0] <= c && c <= kAstralUpper[l][1]) {
return c + kAstralUpper[l][2];
} else {
return c;

2
libc/testlib/testmain.c
View file

@ -86,7 +86,7 @@ int main(int argc, char *argv[]) {
testlib_runalltests();
if (!g_testlib_failed && runbenchmarks_ && weaken(testlib_runallbenchmarks)) {
weaken(testlib_runallbenchmarks)();
if (!g_testlib_failed) {
if (!g_testlib_failed && IsRunningUnderMake()) {
return 254; /* compile.com considers this 0 and propagates output */
}
}

43
libc/zipos/get.c
View file

@ -22,6 +22,7 @@
#include "libc/calls/struct/stat.h"
#include "libc/errno.h"
#include "libc/limits.h"
#include "libc/macros.internal.h"
#include "libc/mem/alloca.h"
#include "libc/runtime/runtime.h"
#include "libc/str/str.h"
@ -33,6 +34,27 @@
#include "libc/zip.h"
#include "libc/zipos/zipos.internal.h"
static uint64_t __zipos_get_min_offset(const uint8_t *base,
const uint8_t *cdir) {
uint64_t i, n, c, r, o;
c = GetZipCdirOffset(cdir);
n = GetZipCdirRecords(cdir);
for (r = c, i = 0; i < n; ++i, c += ZIP_CFILE_HDRSIZE(base + c)) {
o = GetZipCfileOffset(base + c);
if (o < r) r = o;
}
return r;
}
static void __zipos_munmap_unneeded(const uint8_t *base, const uint8_t *cdir,
const uint8_t *map) {
uint64_t n;
n = __zipos_get_min_offset(base, cdir);
n += base - map;
n = ROUNDDOWN(n, FRAMESIZE);
if (n) munmap(map, n);
}
/**
* Returns pointer to zip central directory of current executable.
*/
@ -40,30 +62,31 @@ struct Zipos *__zipos_get(void) {
static bool once;
static struct Zipos zipos;
int fd;
size_t n;
char *path;
size_t size;
sigset_t neu, old;
uint8_t *p, *base, *cdir;
uint8_t *map, *base, *cdir;
if (!once) {
sigfillset(&neu);
sigprocmask(SIG_BLOCK, &neu, &old);
if ((fd = open(program_executable_name, O_RDONLY)) != -1) {
if ((n = getfiledescriptorsize(fd)) != SIZE_MAX &&
(p = mmap(0, n, PROT_READ, MAP_SHARED, fd, 0)) != MAP_FAILED) {
if ((size = getfiledescriptorsize(fd)) != SIZE_MAX &&
(map = mmap(0, size, PROT_READ, MAP_SHARED, fd, 0)) != MAP_FAILED) {
if (endswith(program_executable_name, ".com.dbg")) {
if ((base = memmem(p, n, "MZqFpD", 6))) {
n -= base - p;
if ((base = memmem(map, size, "MZqFpD", 6))) {
size -= base - map;
} else {
base = p;
base = map;
}
} else {
base = p;
base = map;
}
if ((cdir = GetZipCdir(base, n))) {
if ((cdir = GetZipCdir(base, size))) {
__zipos_munmap_unneeded(base, cdir, map);
zipos.map = base;
zipos.cdir = cdir;
} else {
munmap(p, n);
munmap(map, size);
ZTRACE("__zipos_get(%s) → eocd not found", program_executable_name);
}
} else {