mirror of
https://github.com/jart/cosmopolitan.git
synced 2025-07-12 05:59:10 +00:00
Make numerous improvements
- Python static hello world now 1.8mb - Python static fully loaded now 10mb - Python HTTPS client now uses MbedTLS - Python REPL now completes import stmts - Increase stack size for Python for now - Begin synthesizing posixpath and ntpath - Restore Python \N{UNICODE NAME} support - Restore Python NFKD symbol normalization - Add optimized code path for Intel SHA-NI - Get more Python unit tests passing faster - Get Python help() pagination working on NT - Python hashlib now supports MbedTLS PBKDF2 - Make memcpy/memmove/memcmp/bcmp/etc. faster - Add Mersenne Twister and Vigna to LIBC_RAND - Provide privileged __printf() for error code - Fix zipos opendir() so that it reports ENOTDIR - Add basic chmod() implementation for Windows NT - Add Cosmo's best functions to Python cosmo module - Pin function trace indent depth to that of caller - Show memory diagram on invalid access in MODE=dbg - Differentiate stack overflow on crash in MODE=dbg - Add stb_truetype and tools for analyzing font files - Upgrade to UNICODE 13 and reduce its binary footprint - COMPILE.COM now logs resource usage of build commands - Start implementing basic poll() support on bare metal - Set getauxval(AT_EXECFN) to GetModuleFileName() on NT - Add descriptions to strerror() in non-TINY build modes - Add COUNTBRANCH() macro to help with micro-optimizations - Make error / backtrace / asan / memory code more unbreakable - Add fast perfect C implementation of μ-Law and a-Law audio codecs - Make strtol() functions consistent with other libc implementations - Improve Linenoise implementation (see also github.com/jart/bestline) - COMPILE.COM now suppresses stdout/stderr of successful build commands
This commit is contained in:
parent
fa7b4f5bd1
commit
39bf41f4eb
806 changed files with 77494 additions and 63859 deletions
309
libc/intrin/memmove.c
Normal file
309
libc/intrin/memmove.c
Normal file
|
@ -0,0 +1,309 @@
|
|||
/*-*- mode:c;indent-tabs-mode:nil;c-basic-offset:2;tab-width:8;coding:utf-8 -*-│
|
||||
│vi: set net ft=c ts=2 sts=2 sw=2 fenc=utf-8 :vi│
|
||||
╞══════════════════════════════════════════════════════════════════════════════╡
|
||||
│ Copyright 2021 Justine Alexandra Roberts Tunney │
|
||||
│ │
|
||||
│ Permission to use, copy, modify, and/or distribute this software for │
|
||||
│ any purpose with or without fee is hereby granted, provided that the │
|
||||
│ above copyright notice and this permission notice appear in all copies. │
|
||||
│ │
|
||||
│ THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL │
|
||||
│ WARRANTIES WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED │
|
||||
│ WARRANTIES OF MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE │
|
||||
│ AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL │
|
||||
│ DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR │
|
||||
│ PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER │
|
||||
│ TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR │
|
||||
│ PERFORMANCE OF THIS SOFTWARE. │
|
||||
╚─────────────────────────────────────────────────────────────────────────────*/
|
||||
#include "libc/assert.h"
|
||||
#include "libc/dce.h"
|
||||
#include "libc/intrin/asan.internal.h"
|
||||
#include "libc/nexgen32e/nexgen32e.h"
|
||||
#include "libc/nexgen32e/x86feature.h"
|
||||
#include "libc/str/str.h"
|
||||
|
||||
typedef long long xmm_t __attribute__((__vector_size__(16), __aligned__(1)));
|
||||
typedef long long xmm_a __attribute__((__vector_size__(16), __aligned__(16)));
|
||||
|
||||
asm("memcpy = memmove\n\t"
|
||||
".globl\tmemcpy");
|
||||
|
||||
/**
|
||||
* Copies memory.
|
||||
*
|
||||
* memmove n=0 661 picoseconds
|
||||
* memmove n=1 661 ps/byte 1,476 mb/s
|
||||
* memmove n=2 330 ps/byte 2,952 mb/s
|
||||
* memmove n=3 330 ps/byte 2,952 mb/s
|
||||
* memmove n=4 165 ps/byte 5,904 mb/s
|
||||
* memmove n=7 141 ps/byte 6,888 mb/s
|
||||
* memmove n=8 82 ps/byte 11 GB/s
|
||||
* memmove n=15 44 ps/byte 21 GB/s
|
||||
* memmove n=16 41 ps/byte 23 GB/s
|
||||
* memmove n=31 32 ps/byte 29 GB/s
|
||||
* memmove n=32 31 ps/byte 30 GB/s
|
||||
* memmove n=63 21 ps/byte 45 GB/s
|
||||
* memmove n=64 15 ps/byte 61 GB/s
|
||||
* memmove n=127 13 ps/byte 73 GB/s
|
||||
* memmove n=128 31 ps/byte 30 GB/s
|
||||
* memmove n=255 20 ps/byte 45 GB/s
|
||||
* memmove n=256 19 ps/byte 49 GB/s
|
||||
* memmove n=511 16 ps/byte 56 GB/s
|
||||
* memmove n=512 17 ps/byte 54 GB/s
|
||||
* memmove n=1023 18 ps/byte 52 GB/s
|
||||
* memmove n=1024 13 ps/byte 72 GB/s
|
||||
* memmove n=2047 9 ps/byte 96 GB/s
|
||||
* memmove n=2048 9 ps/byte 98 GB/s
|
||||
* memmove n=4095 8 ps/byte 112 GB/s
|
||||
* memmove n=4096 8 ps/byte 109 GB/s
|
||||
* memmove n=8191 7 ps/byte 124 GB/s
|
||||
* memmove n=8192 7 ps/byte 125 GB/s
|
||||
* memmove n=16383 7 ps/byte 134 GB/s
|
||||
* memmove n=16384 7 ps/byte 134 GB/s
|
||||
* memmove n=32767 13 ps/byte 72 GB/s
|
||||
* memmove n=32768 13 ps/byte 72 GB/s
|
||||
* memmove n=65535 13 ps/byte 68 GB/s
|
||||
* memmove n=65536 14 ps/byte 67 GB/s
|
||||
* memmove n=131071 14 ps/byte 65 GB/s
|
||||
* memmove n=131072 14 ps/byte 64 GB/s
|
||||
* memmove n=262143 15 ps/byte 63 GB/s
|
||||
* memmove n=262144 15 ps/byte 63 GB/s
|
||||
* memmove n=524287 15 ps/byte 61 GB/s
|
||||
* memmove n=524288 15 ps/byte 61 GB/s
|
||||
* memmove n=1048575 15 ps/byte 61 GB/s
|
||||
* memmove n=1048576 15 ps/byte 61 GB/s
|
||||
* memmove n=2097151 19 ps/byte 48 GB/s
|
||||
* memmove n=2097152 27 ps/byte 35 GB/s
|
||||
* memmove n=4194303 28 ps/byte 33 GB/s
|
||||
* memmove n=4194304 28 ps/byte 33 GB/s
|
||||
* memmove n=8388607 28 ps/byte 33 GB/s
|
||||
* memmove n=8388608 28 ps/byte 33 GB/s
|
||||
*
|
||||
* DST and SRC may overlap.
|
||||
*
|
||||
* @param dst is destination
|
||||
* @param src is memory to copy
|
||||
* @param n is number of bytes to copy
|
||||
* @return dst
|
||||
* @asyncsignalsafe
|
||||
*/
|
||||
void *memmove(void *dst, const void *src, size_t n) {
|
||||
size_t i;
|
||||
char *d, *r;
|
||||
const char *s;
|
||||
uint64_t a, b;
|
||||
xmm_t v, w, x, y, V, W, X, Y, wut;
|
||||
d = dst;
|
||||
s = src;
|
||||
if (IsTiny()) {
|
||||
if (d <= s) {
|
||||
asm("rep movsb"
|
||||
: "+D"(d), "+S"(s), "+c"(n), "=m"(*(char(*)[n])dst)
|
||||
: "m"(*(char(*)[n])src));
|
||||
} else {
|
||||
d += n - 1;
|
||||
s += n - 1;
|
||||
asm("std\n\t"
|
||||
"rep movsb\n\t"
|
||||
"cld"
|
||||
: "+D"(d), "+S"(s), "+c"(n), "=m"(*(char(*)[n])dst)
|
||||
: "m"(*(char(*)[n])src));
|
||||
}
|
||||
return dst;
|
||||
}
|
||||
switch (n) {
|
||||
case 0:
|
||||
return d;
|
||||
case 1:
|
||||
*d = *s;
|
||||
return d;
|
||||
case 2:
|
||||
__builtin_memcpy(&a, s, 2);
|
||||
__builtin_memcpy(d, &a, 2);
|
||||
return d;
|
||||
case 3:
|
||||
__builtin_memcpy(&a, s, 2);
|
||||
__builtin_memcpy(&b, s + 1, 2);
|
||||
__builtin_memcpy(d, &a, 2);
|
||||
__builtin_memcpy(d + 1, &b, 2);
|
||||
return d;
|
||||
case 4:
|
||||
__builtin_memcpy(&a, s, 4);
|
||||
__builtin_memcpy(d, &a, 4);
|
||||
return d;
|
||||
case 5 ... 7:
|
||||
__builtin_memcpy(&a, s, 4);
|
||||
__builtin_memcpy(&b, s + n - 4, 4);
|
||||
__builtin_memcpy(d, &a, 4);
|
||||
__builtin_memcpy(d + n - 4, &b, 4);
|
||||
return d;
|
||||
case 8:
|
||||
__builtin_memcpy(&a, s, 8);
|
||||
__builtin_memcpy(d, &a, 8);
|
||||
return d;
|
||||
case 9 ... 15:
|
||||
__builtin_memcpy(&a, s, 8);
|
||||
__builtin_memcpy(&b, s + n - 8, 8);
|
||||
__builtin_memcpy(d, &a, 8);
|
||||
__builtin_memcpy(d + n - 8, &b, 8);
|
||||
return d;
|
||||
case 16:
|
||||
*(xmm_t *)d = *(xmm_t *)s;
|
||||
return d;
|
||||
case 17 ... 32:
|
||||
v = *(xmm_t *)s;
|
||||
w = *(xmm_t *)(s + n - 16);
|
||||
*(xmm_t *)d = v;
|
||||
*(xmm_t *)(d + n - 16) = w;
|
||||
return d;
|
||||
case 33 ... 64:
|
||||
v = *(xmm_t *)s;
|
||||
w = *(xmm_t *)(s + 16);
|
||||
x = *(xmm_t *)(s + n - 32);
|
||||
y = *(xmm_t *)(s + n - 16);
|
||||
*(xmm_t *)d = v;
|
||||
*(xmm_t *)(d + 16) = w;
|
||||
*(xmm_t *)(d + n - 32) = x;
|
||||
*(xmm_t *)(d + n - 16) = y;
|
||||
return d;
|
||||
case 65 ... 127:
|
||||
v = *(xmm_t *)s;
|
||||
w = *(xmm_t *)(s + 16);
|
||||
x = *(xmm_t *)(s + 32);
|
||||
y = *(xmm_t *)(s + 48);
|
||||
V = *(xmm_t *)(s + n - 64);
|
||||
W = *(xmm_t *)(s + n - 48);
|
||||
X = *(xmm_t *)(s + n - 32);
|
||||
Y = *(xmm_t *)(s + n - 16);
|
||||
*(xmm_t *)d = v;
|
||||
*(xmm_t *)(d + 16) = w;
|
||||
*(xmm_t *)(d + 32) = x;
|
||||
*(xmm_t *)(d + 48) = y;
|
||||
*(xmm_t *)(d + n - 64) = V;
|
||||
*(xmm_t *)(d + n - 48) = W;
|
||||
*(xmm_t *)(d + n - 32) = X;
|
||||
*(xmm_t *)(d + n - 16) = Y;
|
||||
return d;
|
||||
default:
|
||||
r = d;
|
||||
if (d == s) return d;
|
||||
if (n < kHalfCache3 || !kHalfCache3) {
|
||||
if (d > s) {
|
||||
if (IsAsan() || n < 1024 || !X86_HAVE(ERMS)) {
|
||||
do {
|
||||
n -= 32;
|
||||
v = *(const xmm_t *)(s + n);
|
||||
w = *(const xmm_t *)(s + n + 16);
|
||||
*(xmm_t *)(d + n) = v;
|
||||
*(xmm_t *)(d + n + 16) = w;
|
||||
} while (n >= 32);
|
||||
} else {
|
||||
asm("std\n\t"
|
||||
"rep movsb\n\t"
|
||||
"cld"
|
||||
: "=D"(d), "=S"(s), "+c"(n), "=m"(*(char(*)[n])d)
|
||||
: "0"(d + n - 1), "1"(s + n - 1), "m"(*(char(*)[n])s));
|
||||
return r;
|
||||
}
|
||||
} else {
|
||||
if (IsAsan() || n < 1024 || !X86_HAVE(ERMS)) {
|
||||
i = 0;
|
||||
do {
|
||||
v = *(const xmm_t *)(s + i);
|
||||
w = *(const xmm_t *)(s + i + 16);
|
||||
*(xmm_t *)(d + i) = v;
|
||||
*(xmm_t *)(d + i + 16) = w;
|
||||
} while ((i += 32) + 32 <= n);
|
||||
d += i;
|
||||
s += i;
|
||||
n -= i;
|
||||
} else {
|
||||
asm("rep movsb"
|
||||
: "+D"(d), "+S"(s), "+c"(n), "=m"(*(char(*)[n])d)
|
||||
: "m"(*(char(*)[n])s));
|
||||
return r;
|
||||
}
|
||||
}
|
||||
} else {
|
||||
if (d > s) {
|
||||
while ((uintptr_t)(d + n) & 15) {
|
||||
--n;
|
||||
d[n] = s[n];
|
||||
}
|
||||
do {
|
||||
n -= 32;
|
||||
v = *(const xmm_t *)(s + n);
|
||||
w = *(const xmm_t *)(s + n + 16);
|
||||
__builtin_ia32_movntdq((xmm_a *)(d + n), v);
|
||||
__builtin_ia32_movntdq((xmm_a *)(d + n + 16), w);
|
||||
} while (n >= 32);
|
||||
} else {
|
||||
i = 0;
|
||||
while ((uintptr_t)(d + i) & 15) {
|
||||
d[i] = s[i];
|
||||
++i;
|
||||
}
|
||||
do {
|
||||
v = *(const xmm_t *)(s + i);
|
||||
w = *(const xmm_t *)(s + i + 16);
|
||||
__builtin_ia32_movntdq((xmm_a *)(d + i), v);
|
||||
__builtin_ia32_movntdq((xmm_a *)(d + i + 16), w);
|
||||
} while ((i += 32) + 32 <= n);
|
||||
d += i;
|
||||
s += i;
|
||||
n -= i;
|
||||
}
|
||||
asm("sfence");
|
||||
}
|
||||
switch (n) {
|
||||
case 0:
|
||||
return r;
|
||||
case 17 ... 31:
|
||||
__builtin_memcpy(&v, s, 16);
|
||||
__builtin_memcpy(&w, s + n - 16, 16);
|
||||
__builtin_memcpy(d, &v, 16);
|
||||
__builtin_memcpy(d + n - 16, &w, 16);
|
||||
return r;
|
||||
case 9 ... 15:
|
||||
__builtin_memcpy(&a, s, 8);
|
||||
__builtin_memcpy(&b, s + n - 8, 8);
|
||||
__builtin_memcpy(d, &a, 8);
|
||||
__builtin_memcpy(d + n - 8, &b, 8);
|
||||
return r;
|
||||
case 5 ... 7:
|
||||
__builtin_memcpy(&a, s, 4);
|
||||
__builtin_memcpy(&b, s + n - 4, 4);
|
||||
__builtin_memcpy(d, &a, 4);
|
||||
__builtin_memcpy(d + n - 4, &b, 4);
|
||||
return r;
|
||||
case 16:
|
||||
__builtin_memcpy(&v, s, 16);
|
||||
__builtin_memcpy(d, &v, 16);
|
||||
return r;
|
||||
case 8:
|
||||
__builtin_memcpy(&a, s, 8);
|
||||
__builtin_memcpy(d, &a, 8);
|
||||
return r;
|
||||
case 4:
|
||||
__builtin_memcpy(&a, s, 4);
|
||||
__builtin_memcpy(d, &a, 4);
|
||||
return r;
|
||||
case 1:
|
||||
*d = *s;
|
||||
return r;
|
||||
case 2:
|
||||
__builtin_memcpy(&a, s, 2);
|
||||
__builtin_memcpy(d, &a, 2);
|
||||
return r;
|
||||
case 3:
|
||||
__builtin_memcpy(&a, s, 2);
|
||||
__builtin_memcpy(&b, s + 1, 2);
|
||||
__builtin_memcpy(d, &a, 2);
|
||||
__builtin_memcpy(d + 1, &b, 2);
|
||||
return r;
|
||||
default:
|
||||
unreachable;
|
||||
}
|
||||
}
|
||||
}
|
Loading…
Add table
Add a link
Reference in a new issue