mirror of
https://github.com/jart/cosmopolitan.git
synced 2025-02-07 15:03:34 +00:00
97b7116953
After going through the MODE=dbg and MODE=zero build modes, a bunch of little issues were identified, which have been addressed. Fixing those issues created even more troubles for the project, because it improved our ability to detect latent problems which are getting fixed so fast.
812 lines
32 KiB
C
812 lines
32 KiB
C
// clang-format off
|
|
#ifdef __x86_64__
|
|
|
|
/*
|
|
* xxHash - Extremely Fast Hash algorithm
|
|
* Copyright (C) 2020-2021 Yann Collet
|
|
*
|
|
* BSD 2-Clause License (https://www.opensource.org/licenses/bsd-license.php)
|
|
*
|
|
* Redistribution and use in source and binary forms, with or without
|
|
* modification, are permitted provided that the following conditions are
|
|
* met:
|
|
*
|
|
* * Redistributions of source code must retain the above copyright
|
|
* notice, this list of conditions and the following disclaimer.
|
|
* * Redistributions in binary form must reproduce the above
|
|
* copyright notice, this list of conditions and the following disclaimer
|
|
* in the documentation and/or other materials provided with the
|
|
* distribution.
|
|
*
|
|
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
|
|
* "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
|
|
* LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
|
|
* A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
|
|
* OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
|
|
* SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
|
|
* LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
|
|
* DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
|
|
* THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
|
|
* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
|
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
|
*
|
|
* You can contact the author at:
|
|
* - xxHash homepage: https://www.xxhash.com
|
|
* - xxHash source repository: https://github.com/Cyan4973/xxHash
|
|
*/
|
|
|
|
|
|
/*!
|
|
* @file xxh_x86dispatch.c
|
|
*
|
|
* Automatic dispatcher code for the @ref XXH3_family on x86-based targets.
|
|
*
|
|
* Optional add-on.
|
|
*
|
|
* **Compile this file with the default flags for your target.** Do not compile
|
|
* with flags like `-mavx*`, `-march=native`, or `/arch:AVX*`, there will be
|
|
* an error. See @ref XXH_X86DISPATCH_ALLOW_AVX for details.
|
|
*
|
|
* @defgroup dispatch x86 Dispatcher
|
|
* @{
|
|
*/
|
|
|
|
#if defined (__cplusplus)
|
|
extern "C" {
|
|
#endif
|
|
|
|
#if !(defined(__x86_64__) || defined(__i386__) || defined(_M_IX86) || defined(_M_X64))
|
|
# error "Dispatching is currently only supported on x86 and x86_64."
|
|
#endif
|
|
|
|
/*!
|
|
* @def XXH_X86DISPATCH_ALLOW_AVX
|
|
* @brief Disables the AVX sanity check.
|
|
*
|
|
* xxh_x86dispatch.c is intended to be compiled for the minimum target, and
|
|
* it selectively enables SSE2, AVX2, and AVX512 when it is needed.
|
|
*
|
|
* Compiling with options like `-mavx*`, `-march=native`, or `/arch:AVX*`
|
|
* _globally_ will always enable this feature, and therefore makes it
|
|
* undefined behavior to execute on any CPU without said feature.
|
|
*
|
|
* Even if the source code isn't directly using AVX intrinsics in a function,
|
|
* the compiler can still generate AVX code from autovectorization and by
|
|
* "upgrading" SSE2 intrinsics to use the VEX prefixes (a.k.a. AVX128).
|
|
*
|
|
* Define XXH_X86DISPATCH_ALLOW_AVX to ignore this check,
|
|
* thus accepting that the produced binary will not work correctly
|
|
* on any CPU with less features than the ones stated at compilation time.
|
|
*/
|
|
#ifdef XXH_DOXYGEN
|
|
# define XXH_X86DISPATCH_ALLOW_AVX
|
|
#endif
|
|
|
|
#if defined(__AVX__) && !defined(XXH_X86DISPATCH_ALLOW_AVX)
|
|
# error "Error: if xxh_x86dispatch.c is compiled with AVX enabled, the resulting binary will crash on sse2-only cpus !! " \
|
|
"If you nonetheless want to do that, please enable the XXH_X86DISPATCH_ALLOW_AVX build variable"
|
|
#endif
|
|
|
|
#ifdef __has_include
|
|
# define XXH_HAS_INCLUDE(header) __has_include(header)
|
|
#else
|
|
# define XXH_HAS_INCLUDE(header) 0
|
|
#endif
|
|
|
|
/*!
|
|
* @def XXH_DISPATCH_SCALAR
|
|
* @brief Enables/dispatching the scalar code path.
|
|
*
|
|
* If this is defined to 0, SSE2 support is assumed. This reduces code size
|
|
* when the scalar path is not needed.
|
|
*
|
|
* This is automatically defined to 0 when...
|
|
* - SSE2 support is enabled in the compiler
|
|
* - Targeting x86_64
|
|
* - Targeting Android x86
|
|
* - Targeting macOS
|
|
*/
|
|
#ifndef XXH_DISPATCH_SCALAR
|
|
# if defined(__SSE2__) || (defined(_M_IX86_FP) && _M_IX86_FP >= 2) /* SSE2 on by default */ \
|
|
|| defined(__x86_64__) || defined(_M_X64) /* x86_64 */ \
|
|
|| defined(__ANDROID__) || defined(__APPLEv__) /* Android or macOS */
|
|
# define XXH_DISPATCH_SCALAR 0 /* disable */
|
|
# else
|
|
# define XXH_DISPATCH_SCALAR 1
|
|
# endif
|
|
#endif
|
|
/*!
|
|
* @def XXH_DISPATCH_AVX2
|
|
* @brief Enables/disables dispatching for AVX2.
|
|
*
|
|
* This is automatically detected if it is not defined.
|
|
* - GCC 4.7 and later are known to support AVX2, but >4.9 is required for
|
|
* to get the AVX2 intrinsics and typedefs without -mavx -mavx2.
|
|
* - Visual Studio 2013 Update 2 and later are known to support AVX2.
|
|
* - The GCC/Clang internal header `<avx2intrin.h>` is detected. While this is
|
|
* not allowed to be included directly, it still appears in the builtin
|
|
* include path and is detectable with `__has_include`.
|
|
*
|
|
* @see XXH_AVX2
|
|
*/
|
|
#ifndef XXH_DISPATCH_AVX2
|
|
# if (defined(__GNUC__) && (__GNUC__ > 4)) /* GCC 5.0+ */ \
|
|
|| (defined(_MSC_VER) && _MSC_VER >= 1900) /* VS 2015+ */ \
|
|
|| (defined(_MSC_FULL_VER) && _MSC_FULL_VER >= 180030501) /* VS 2013 Update 2 */ \
|
|
|| XXH_HAS_INCLUDE(<avx2intrin.h>) /* GCC/Clang internal header */
|
|
# define XXH_DISPATCH_AVX2 1 /* enable dispatch towards AVX2 */
|
|
# else
|
|
# define XXH_DISPATCH_AVX2 0
|
|
# endif
|
|
#endif /* XXH_DISPATCH_AVX2 */
|
|
|
|
/*!
|
|
* @def XXH_DISPATCH_AVX512
|
|
* @brief Enables/disables dispatching for AVX512.
|
|
*
|
|
* Automatically detected if one of the following conditions is met:
|
|
* - GCC 4.9 and later are known to support AVX512.
|
|
* - Visual Studio 2017 and later are known to support AVX2.
|
|
* - The GCC/Clang internal header `<avx512fintrin.h>` is detected. While this
|
|
* is not allowed to be included directly, it still appears in the builtin
|
|
* include path and is detectable with `__has_include`.
|
|
*
|
|
* @see XXH_AVX512
|
|
*/
|
|
#ifndef XXH_DISPATCH_AVX512
|
|
# if (defined(__GNUC__) \
|
|
&& (__GNUC__ > 4 || (__GNUC__ == 4 && __GNUC_MINOR__ >= 9))) /* GCC 4.9+ */ \
|
|
|| (defined(_MSC_VER) && _MSC_VER >= 1910) /* VS 2017+ */ \
|
|
|| XXH_HAS_INCLUDE(<avx512fintrin.h>) /* GCC/Clang internal header */
|
|
# define XXH_DISPATCH_AVX512 1 /* enable dispatch towards AVX512 */
|
|
# else
|
|
# define XXH_DISPATCH_AVX512 0
|
|
# endif
|
|
#endif /* XXH_DISPATCH_AVX512 */
|
|
|
|
/*!
|
|
* @def XXH_TARGET_SSE2
|
|
* @brief Allows a function to be compiled with SSE2 intrinsics.
|
|
*
|
|
* Uses `__attribute__((__target__("sse2")))` on GCC to allow SSE2 to be used
|
|
* even with `-mno-sse2`.
|
|
*
|
|
* @def XXH_TARGET_AVX2
|
|
* @brief Like @ref XXH_TARGET_SSE2, but for AVX2.
|
|
*
|
|
* @def XXH_TARGET_AVX512
|
|
* @brief Like @ref XXH_TARGET_SSE2, but for AVX512.
|
|
*/
|
|
#if defined(__GNUC__)
|
|
#include "third_party/intel/emmintrin.internal.h" /* SSE2 */
|
|
# if XXH_DISPATCH_AVX2 || XXH_DISPATCH_AVX512
|
|
#include "third_party/intel/immintrin.internal.h" /* AVX2, AVX512F */
|
|
# endif
|
|
# define XXH_TARGET_SSE2 __attribute__((__target__("sse2")))
|
|
# define XXH_TARGET_AVX2 __attribute__((__target__("avx2")))
|
|
# define XXH_TARGET_AVX512 __attribute__((__target__("avx512f")))
|
|
#elif defined(_MSC_VER)
|
|
// MISSING #include <intrin.h>
|
|
# define XXH_TARGET_SSE2
|
|
# define XXH_TARGET_AVX2
|
|
# define XXH_TARGET_AVX512
|
|
#else
|
|
# error "Dispatching is currently not supported for your compiler."
|
|
#endif
|
|
|
|
#ifdef XXH_DISPATCH_DEBUG
|
|
/* debug logging */
|
|
#include "libc/calls/calls.h"
|
|
#include "libc/calls/weirdtypes.h"
|
|
#include "libc/fmt/fmt.h"
|
|
#include "libc/stdio/dprintf.h"
|
|
#include "libc/stdio/stdio.h"
|
|
#include "libc/stdio/temp.h"
|
|
#include "third_party/musl/tempnam.h"
|
|
# define XXH_debugPrint(str) { fprintf(stderr, "DEBUG: xxHash dispatch: %s \n", str); fflush(NULL); }
|
|
#else
|
|
# define XXH_debugPrint(str) ((void)0)
|
|
# undef NDEBUG /* avoid redefinition */
|
|
# define NDEBUG
|
|
#endif
|
|
#include "libc/assert.h"
|
|
|
|
#define XXH_INLINE_ALL
|
|
#define XXH_X86DISPATCH
|
|
#include "third_party/xxhash/xxhash.h"
|
|
|
|
#ifndef XXH_HAS_ATTRIBUTE
|
|
# ifdef __has_attribute
|
|
# define XXH_HAS_ATTRIBUTE(...) __has_attribute(__VA_ARGS__)
|
|
# else
|
|
# define XXH_HAS_ATTRIBUTE(...) 0
|
|
# endif
|
|
#endif
|
|
|
|
#if XXH_HAS_ATTRIBUTE(constructor)
|
|
# define XXH_CONSTRUCTOR __attribute__((constructor))
|
|
# define XXH_DISPATCH_MAYBE_NULL 0
|
|
#else
|
|
# define XXH_CONSTRUCTOR
|
|
# define XXH_DISPATCH_MAYBE_NULL 1
|
|
#endif
|
|
|
|
|
|
/*
|
|
* Support both AT&T and Intel dialects
|
|
*
|
|
* GCC doesn't convert AT&T syntax to Intel syntax, and will error out if
|
|
* compiled with -masm=intel. Instead, it supports dialect switching with
|
|
* curly braces: { AT&T syntax | Intel syntax }
|
|
*
|
|
* Clang's integrated assembler automatically converts AT&T syntax to Intel if
|
|
* needed, making the dialect switching useless (it isn't even supported).
|
|
*
|
|
* Note: Comments are written in the inline assembly itself.
|
|
*/
|
|
#ifdef __clang__
|
|
# define XXH_I_ATT(intel, att) att "\n\t"
|
|
#else
|
|
# define XXH_I_ATT(intel, att) "{" att "|" intel "}\n\t"
|
|
#endif
|
|
|
|
/*!
|
|
* @internal
|
|
* @brief Runs CPUID.
|
|
*
|
|
* @param eax , ecx The parameters to pass to CPUID, %eax and %ecx respectively.
|
|
* @param abcd The array to store the result in, `{ eax, ebx, ecx, edx }`
|
|
*/
|
|
static void XXH_cpuid(xxh_u32 eax, xxh_u32 ecx, xxh_u32* abcd)
|
|
{
|
|
#if defined(_MSC_VER)
|
|
__cpuidex(abcd, eax, ecx);
|
|
#else
|
|
xxh_u32 ebx, edx;
|
|
# if defined(__i386__) && defined(__PIC__)
|
|
__asm__(
|
|
"# Call CPUID\n\t"
|
|
"#\n\t"
|
|
"# On 32-bit x86 with PIC enabled, we are not allowed to overwrite\n\t"
|
|
"# EBX, so we use EDI instead.\n\t"
|
|
XXH_I_ATT("mov edi, ebx", "movl %%ebx, %%edi")
|
|
XXH_I_ATT("cpuid", "cpuid" )
|
|
XXH_I_ATT("xchg edi, ebx", "xchgl %%ebx, %%edi")
|
|
: "=D" (ebx),
|
|
# else
|
|
__asm__(
|
|
"# Call CPUID\n\t"
|
|
XXH_I_ATT("cpuid", "cpuid")
|
|
: "=b" (ebx),
|
|
# endif
|
|
"+a" (eax), "+c" (ecx), "=d" (edx));
|
|
abcd[0] = eax;
|
|
abcd[1] = ebx;
|
|
abcd[2] = ecx;
|
|
abcd[3] = edx;
|
|
#endif
|
|
}
|
|
|
|
/*
|
|
* Modified version of Intel's guide
|
|
* https://software.intel.com/en-us/articles/how-to-detect-new-instruction-support-in-the-4th-generation-intel-core-processor-family
|
|
*/
|
|
|
|
#if XXH_DISPATCH_AVX2 || XXH_DISPATCH_AVX512
|
|
/*!
|
|
* @internal
|
|
* @brief Runs `XGETBV`.
|
|
*
|
|
* While the CPU may support AVX2, the operating system might not properly save
|
|
* the full YMM/ZMM registers.
|
|
*
|
|
* xgetbv is used for detecting this: Any compliant operating system will define
|
|
* a set of flags in the xcr0 register indicating how it saves the AVX registers.
|
|
*
|
|
* You can manually disable this flag on Windows by running, as admin:
|
|
*
|
|
* bcdedit.exe /set xsavedisable 1
|
|
*
|
|
* and rebooting. Run the same command with 0 to re-enable it.
|
|
*/
|
|
static xxh_u64 XXH_xgetbv(void)
|
|
{
|
|
#if defined(_MSC_VER)
|
|
return _xgetbv(0); /* min VS2010 SP1 compiler is required */
|
|
#else
|
|
xxh_u32 xcr0_lo, xcr0_hi;
|
|
__asm__(
|
|
"# Call XGETBV\n\t"
|
|
"#\n\t"
|
|
"# Older assemblers (e.g. macOS's ancient GAS version) don't support\n\t"
|
|
"# the XGETBV opcode, so we encode it by hand instead.\n\t"
|
|
"# See <https://github.com/asmjit/asmjit/issues/78> for details.\n\t"
|
|
".byte 0x0f, 0x01, 0xd0\n\t"
|
|
: "=a" (xcr0_lo), "=d" (xcr0_hi) : "c" (0));
|
|
return xcr0_lo | ((xxh_u64)xcr0_hi << 32);
|
|
#endif
|
|
}
|
|
#endif
|
|
|
|
#define XXH_SSE2_CPUID_MASK (1 << 26)
|
|
#define XXH_OSXSAVE_CPUID_MASK ((1 << 26) | (1 << 27))
|
|
#define XXH_AVX2_CPUID_MASK (1 << 5)
|
|
#define XXH_AVX2_XGETBV_MASK ((1 << 2) | (1 << 1))
|
|
#define XXH_AVX512F_CPUID_MASK (1 << 16)
|
|
#define XXH_AVX512F_XGETBV_MASK ((7 << 5) | (1 << 2) | (1 << 1))
|
|
|
|
/*!
|
|
* @internal
|
|
* @brief Returns the best XXH3 implementation.
|
|
*
|
|
* Runs various CPUID/XGETBV tests to try and determine the best implementation.
|
|
*
|
|
* @return The best @ref XXH_VECTOR implementation.
|
|
* @see XXH_VECTOR_TYPES
|
|
*/
|
|
static int XXH_featureTest(void)
|
|
{
|
|
xxh_u32 abcd[4];
|
|
xxh_u32 max_leaves;
|
|
int best = XXH_SCALAR;
|
|
#if XXH_DISPATCH_AVX2 || XXH_DISPATCH_AVX512
|
|
xxh_u64 xgetbv_val;
|
|
#endif
|
|
#if defined(__GNUC__) && defined(__i386__)
|
|
xxh_u32 cpuid_supported;
|
|
__asm__(
|
|
"# For the sake of ruthless backwards compatibility, check if CPUID\n\t"
|
|
"# is supported in the EFLAGS on i386.\n\t"
|
|
"# This is not necessary on x86_64 - CPUID is mandatory.\n\t"
|
|
"# The ID flag (bit 21) in the EFLAGS register indicates support\n\t"
|
|
"# for the CPUID instruction. If a software procedure can set and\n\t"
|
|
"# clear this flag, the processor executing the procedure supports\n\t"
|
|
"# the CPUID instruction.\n\t"
|
|
"# <https://c9x.me/x86/html/file_module_x86_id_45.html>\n\t"
|
|
"#\n\t"
|
|
"# Routine is from <https://wiki.osdev.org/CPUID>.\n\t"
|
|
|
|
"# Save EFLAGS\n\t"
|
|
XXH_I_ATT("pushfd", "pushfl" )
|
|
"# Store EFLAGS\n\t"
|
|
XXH_I_ATT("pushfd", "pushfl" )
|
|
"# Invert the ID bit in stored EFLAGS\n\t"
|
|
XXH_I_ATT("xor dword ptr[esp], 0x200000", "xorl $0x200000, (%%esp)")
|
|
"# Load stored EFLAGS (with ID bit inverted)\n\t"
|
|
XXH_I_ATT("popfd", "popfl" )
|
|
"# Store EFLAGS again (ID bit may or not be inverted)\n\t"
|
|
XXH_I_ATT("pushfd", "pushfl" )
|
|
"# eax = modified EFLAGS (ID bit may or may not be inverted)\n\t"
|
|
XXH_I_ATT("pop eax", "popl %%eax" )
|
|
"# eax = whichever bits were changed\n\t"
|
|
XXH_I_ATT("xor eax, dword ptr[esp]", "xorl (%%esp), %%eax" )
|
|
"# Restore original EFLAGS\n\t"
|
|
XXH_I_ATT("popfd", "popfl" )
|
|
"# eax = zero if ID bit can't be changed, else non-zero\n\t"
|
|
XXH_I_ATT("and eax, 0x200000", "andl $0x200000, %%eax" )
|
|
: "=a" (cpuid_supported) :: "cc");
|
|
|
|
if (XXH_unlikely(!cpuid_supported)) {
|
|
XXH_debugPrint("CPUID support is not detected!");
|
|
return best;
|
|
}
|
|
|
|
#endif
|
|
/* Check how many CPUID pages we have */
|
|
XXH_cpuid(0, 0, abcd);
|
|
max_leaves = abcd[0];
|
|
|
|
/* Shouldn't happen on hardware, but happens on some QEMU configs. */
|
|
if (XXH_unlikely(max_leaves == 0)) {
|
|
XXH_debugPrint("Max CPUID leaves == 0!");
|
|
return best;
|
|
}
|
|
|
|
/* Check for SSE2, OSXSAVE and xgetbv */
|
|
XXH_cpuid(1, 0, abcd);
|
|
|
|
/*
|
|
* Test for SSE2. The check is redundant on x86_64, but it doesn't hurt.
|
|
*/
|
|
if (XXH_unlikely((abcd[3] & XXH_SSE2_CPUID_MASK) != XXH_SSE2_CPUID_MASK))
|
|
return best;
|
|
|
|
XXH_debugPrint("SSE2 support detected.");
|
|
|
|
best = XXH_SSE2;
|
|
#if XXH_DISPATCH_AVX2 || XXH_DISPATCH_AVX512
|
|
/* Make sure we have enough leaves */
|
|
if (XXH_unlikely(max_leaves < 7))
|
|
return best;
|
|
|
|
/* Test for OSXSAVE and XGETBV */
|
|
if ((abcd[2] & XXH_OSXSAVE_CPUID_MASK) != XXH_OSXSAVE_CPUID_MASK)
|
|
return best;
|
|
|
|
/* CPUID check for AVX features */
|
|
XXH_cpuid(7, 0, abcd);
|
|
|
|
xgetbv_val = XXH_xgetbv();
|
|
#if XXH_DISPATCH_AVX2
|
|
/* Validate that AVX2 is supported by the CPU */
|
|
if ((abcd[1] & XXH_AVX2_CPUID_MASK) != XXH_AVX2_CPUID_MASK)
|
|
return best;
|
|
|
|
/* Validate that the OS supports YMM registers */
|
|
if ((xgetbv_val & XXH_AVX2_XGETBV_MASK) != XXH_AVX2_XGETBV_MASK) {
|
|
XXH_debugPrint("AVX2 supported by the CPU, but not the OS.");
|
|
return best;
|
|
}
|
|
|
|
/* AVX2 supported */
|
|
XXH_debugPrint("AVX2 support detected.");
|
|
best = XXH_AVX2;
|
|
#endif
|
|
#if XXH_DISPATCH_AVX512
|
|
/* Check if AVX512F is supported by the CPU */
|
|
if ((abcd[1] & XXH_AVX512F_CPUID_MASK) != XXH_AVX512F_CPUID_MASK) {
|
|
XXH_debugPrint("AVX512F not supported by CPU");
|
|
return best;
|
|
}
|
|
|
|
/* Validate that the OS supports ZMM registers */
|
|
if ((xgetbv_val & XXH_AVX512F_XGETBV_MASK) != XXH_AVX512F_XGETBV_MASK) {
|
|
XXH_debugPrint("AVX512F supported by the CPU, but not the OS.");
|
|
return best;
|
|
}
|
|
|
|
/* AVX512F supported */
|
|
XXH_debugPrint("AVX512F support detected.");
|
|
best = XXH_AVX512;
|
|
#endif
|
|
#endif
|
|
return best;
|
|
}
|
|
|
|
|
|
/* === Vector implementations === */
|
|
|
|
/*!
|
|
* @internal
|
|
* @brief Defines the various dispatch functions.
|
|
*
|
|
* TODO: Consolidate?
|
|
*
|
|
* @param suffix The suffix for the functions, e.g. sse2 or scalar
|
|
* @param target XXH_TARGET_* or empty.
|
|
*/
|
|
#define XXH_DEFINE_DISPATCH_FUNCS(suffix, target) \
|
|
\
|
|
/* === XXH3, default variants === */ \
|
|
\
|
|
XXH_NO_INLINE target XXH64_hash_t \
|
|
XXHL64_default_##suffix(XXH_NOESCAPE const void* XXH_RESTRICT input, \
|
|
size_t len) \
|
|
{ \
|
|
return XXH3_hashLong_64b_internal( \
|
|
input, len, XXH3_kSecret, sizeof(XXH3_kSecret), \
|
|
XXH3_accumulate_##suffix, XXH3_scrambleAcc_##suffix \
|
|
); \
|
|
} \
|
|
\
|
|
/* === XXH3, Seeded variants === */ \
|
|
\
|
|
XXH_NO_INLINE target XXH64_hash_t \
|
|
XXHL64_seed_##suffix(XXH_NOESCAPE const void* XXH_RESTRICT input, size_t len, \
|
|
XXH64_hash_t seed) \
|
|
{ \
|
|
return XXH3_hashLong_64b_withSeed_internal( \
|
|
input, len, seed, XXH3_accumulate_##suffix, \
|
|
XXH3_scrambleAcc_##suffix, XXH3_initCustomSecret_##suffix \
|
|
); \
|
|
} \
|
|
\
|
|
/* === XXH3, Secret variants === */ \
|
|
\
|
|
XXH_NO_INLINE target XXH64_hash_t \
|
|
XXHL64_secret_##suffix(XXH_NOESCAPE const void* XXH_RESTRICT input, \
|
|
size_t len, XXH_NOESCAPE const void* secret, \
|
|
size_t secretLen) \
|
|
{ \
|
|
return XXH3_hashLong_64b_internal( \
|
|
input, len, secret, secretLen, \
|
|
XXH3_accumulate_##suffix, XXH3_scrambleAcc_##suffix \
|
|
); \
|
|
} \
|
|
\
|
|
/* === XXH3 update variants === */ \
|
|
\
|
|
XXH_NO_INLINE target XXH_errorcode \
|
|
XXH3_update_##suffix(XXH_NOESCAPE XXH3_state_t* state, \
|
|
XXH_NOESCAPE const void* input, size_t len) \
|
|
{ \
|
|
return XXH3_update(state, (const xxh_u8*)input, len, \
|
|
XXH3_accumulate_##suffix, XXH3_scrambleAcc_##suffix); \
|
|
} \
|
|
\
|
|
/* === XXH128 default variants === */ \
|
|
\
|
|
XXH_NO_INLINE target XXH128_hash_t \
|
|
XXHL128_default_##suffix(XXH_NOESCAPE const void* XXH_RESTRICT input, \
|
|
size_t len) \
|
|
{ \
|
|
return XXH3_hashLong_128b_internal( \
|
|
input, len, XXH3_kSecret, sizeof(XXH3_kSecret), \
|
|
XXH3_accumulate_##suffix, XXH3_scrambleAcc_##suffix \
|
|
); \
|
|
} \
|
|
\
|
|
/* === XXH128 Secret variants === */ \
|
|
\
|
|
XXH_NO_INLINE target XXH128_hash_t \
|
|
XXHL128_secret_##suffix(XXH_NOESCAPE const void* XXH_RESTRICT input, \
|
|
size_t len, \
|
|
XXH_NOESCAPE const void* XXH_RESTRICT secret, \
|
|
size_t secretLen) \
|
|
{ \
|
|
return XXH3_hashLong_128b_internal( \
|
|
input, len, (const xxh_u8*)secret, secretLen, \
|
|
XXH3_accumulate_##suffix, XXH3_scrambleAcc_##suffix); \
|
|
} \
|
|
\
|
|
/* === XXH128 Seeded variants === */ \
|
|
\
|
|
XXH_NO_INLINE target XXH128_hash_t \
|
|
XXHL128_seed_##suffix(XXH_NOESCAPE const void* XXH_RESTRICT input, size_t len,\
|
|
XXH64_hash_t seed) \
|
|
{ \
|
|
return XXH3_hashLong_128b_withSeed_internal(input, len, seed, \
|
|
XXH3_accumulate_##suffix, XXH3_scrambleAcc_##suffix, \
|
|
XXH3_initCustomSecret_##suffix); \
|
|
}
|
|
|
|
/* End XXH_DEFINE_DISPATCH_FUNCS */
|
|
|
|
#if XXH_DISPATCH_SCALAR
|
|
XXH_DEFINE_DISPATCH_FUNCS(scalar, /* nothing */)
|
|
#endif
|
|
XXH_DEFINE_DISPATCH_FUNCS(sse2, XXH_TARGET_SSE2)
|
|
#if XXH_DISPATCH_AVX2
|
|
XXH_DEFINE_DISPATCH_FUNCS(avx2, XXH_TARGET_AVX2)
|
|
#endif
|
|
#if XXH_DISPATCH_AVX512
|
|
XXH_DEFINE_DISPATCH_FUNCS(avx512, XXH_TARGET_AVX512)
|
|
#endif
|
|
#undef XXH_DEFINE_DISPATCH_FUNCS
|
|
|
|
/* ==== Dispatchers ==== */
|
|
|
|
typedef XXH64_hash_t (*XXH3_dispatchx86_hashLong64_default)(XXH_NOESCAPE const void* XXH_RESTRICT, size_t);
|
|
|
|
typedef XXH64_hash_t (*XXH3_dispatchx86_hashLong64_withSeed)(XXH_NOESCAPE const void* XXH_RESTRICT, size_t, XXH64_hash_t);
|
|
|
|
typedef XXH64_hash_t (*XXH3_dispatchx86_hashLong64_withSecret)(XXH_NOESCAPE const void* XXH_RESTRICT, size_t, XXH_NOESCAPE const void* XXH_RESTRICT, size_t);
|
|
|
|
typedef XXH_errorcode (*XXH3_dispatchx86_update)(XXH_NOESCAPE XXH3_state_t*, XXH_NOESCAPE const void*, size_t);
|
|
|
|
typedef struct {
|
|
XXH3_dispatchx86_hashLong64_default hashLong64_default;
|
|
XXH3_dispatchx86_hashLong64_withSeed hashLong64_seed;
|
|
XXH3_dispatchx86_hashLong64_withSecret hashLong64_secret;
|
|
XXH3_dispatchx86_update update;
|
|
} XXH_dispatchFunctions_s;
|
|
|
|
#define XXH_NB_DISPATCHES 4
|
|
|
|
/*!
|
|
* @internal
|
|
* @brief Table of dispatchers for @ref XXH3_64bits().
|
|
*
|
|
* @pre The indices must match @ref XXH_VECTOR_TYPE.
|
|
*/
|
|
static const XXH_dispatchFunctions_s XXH_kDispatch[XXH_NB_DISPATCHES] = {
|
|
#if XXH_DISPATCH_SCALAR
|
|
/* Scalar */ { XXHL64_default_scalar, XXHL64_seed_scalar, XXHL64_secret_scalar, XXH3_update_scalar },
|
|
#else
|
|
/* Scalar */ { NULL, NULL, NULL, NULL },
|
|
#endif
|
|
/* SSE2 */ { XXHL64_default_sse2, XXHL64_seed_sse2, XXHL64_secret_sse2, XXH3_update_sse2 },
|
|
#if XXH_DISPATCH_AVX2
|
|
/* AVX2 */ { XXHL64_default_avx2, XXHL64_seed_avx2, XXHL64_secret_avx2, XXH3_update_avx2 },
|
|
#else
|
|
/* AVX2 */ { NULL, NULL, NULL, NULL },
|
|
#endif
|
|
#if XXH_DISPATCH_AVX512
|
|
/* AVX512 */ { XXHL64_default_avx512, XXHL64_seed_avx512, XXHL64_secret_avx512, XXH3_update_avx512 }
|
|
#else
|
|
/* AVX512 */ { NULL, NULL, NULL, NULL }
|
|
#endif
|
|
};
|
|
/*!
|
|
* @internal
|
|
* @brief The selected dispatch table for @ref XXH3_64bits().
|
|
*/
|
|
static XXH_dispatchFunctions_s XXH_g_dispatch = { NULL, NULL, NULL, NULL };
|
|
|
|
|
|
typedef XXH128_hash_t (*XXH3_dispatchx86_hashLong128_default)(XXH_NOESCAPE const void* XXH_RESTRICT, size_t);
|
|
|
|
typedef XXH128_hash_t (*XXH3_dispatchx86_hashLong128_withSeed)(XXH_NOESCAPE const void* XXH_RESTRICT, size_t, XXH64_hash_t);
|
|
|
|
typedef XXH128_hash_t (*XXH3_dispatchx86_hashLong128_withSecret)(XXH_NOESCAPE const void* XXH_RESTRICT, size_t, const void* XXH_RESTRICT, size_t);
|
|
|
|
typedef struct {
|
|
XXH3_dispatchx86_hashLong128_default hashLong128_default;
|
|
XXH3_dispatchx86_hashLong128_withSeed hashLong128_seed;
|
|
XXH3_dispatchx86_hashLong128_withSecret hashLong128_secret;
|
|
XXH3_dispatchx86_update update;
|
|
} XXH_dispatch128Functions_s;
|
|
|
|
|
|
/*!
|
|
* @internal
|
|
* @brief Table of dispatchers for @ref XXH3_128bits().
|
|
*
|
|
* @pre The indices must match @ref XXH_VECTOR_TYPE.
|
|
*/
|
|
static const XXH_dispatch128Functions_s XXH_kDispatch128[XXH_NB_DISPATCHES] = {
|
|
#if XXH_DISPATCH_SCALAR
|
|
/* Scalar */ { XXHL128_default_scalar, XXHL128_seed_scalar, XXHL128_secret_scalar, XXH3_update_scalar },
|
|
#else
|
|
/* Scalar */ { NULL, NULL, NULL, NULL },
|
|
#endif
|
|
/* SSE2 */ { XXHL128_default_sse2, XXHL128_seed_sse2, XXHL128_secret_sse2, XXH3_update_sse2 },
|
|
#if XXH_DISPATCH_AVX2
|
|
/* AVX2 */ { XXHL128_default_avx2, XXHL128_seed_avx2, XXHL128_secret_avx2, XXH3_update_avx2 },
|
|
#else
|
|
/* AVX2 */ { NULL, NULL, NULL, NULL },
|
|
#endif
|
|
#if XXH_DISPATCH_AVX512
|
|
/* AVX512 */ { XXHL128_default_avx512, XXHL128_seed_avx512, XXHL128_secret_avx512, XXH3_update_avx512 }
|
|
#else
|
|
/* AVX512 */ { NULL, NULL, NULL, NULL }
|
|
#endif
|
|
};
|
|
|
|
/*!
|
|
* @internal
|
|
* @brief The selected dispatch table for @ref XXH3_64bits().
|
|
*/
|
|
static XXH_dispatch128Functions_s XXH_g_dispatch128 = { NULL, NULL, NULL, NULL };
|
|
|
|
/*!
|
|
* @internal
|
|
* @brief Runs a CPUID check and sets the correct dispatch tables.
|
|
*/
|
|
static XXH_CONSTRUCTOR void XXH_setDispatch(void)
|
|
{
|
|
int vecID = XXH_featureTest();
|
|
XXH_STATIC_ASSERT(XXH_AVX512 == XXH_NB_DISPATCHES-1);
|
|
assert(XXH_SCALAR <= vecID && vecID <= XXH_AVX512);
|
|
#if !XXH_DISPATCH_SCALAR
|
|
assert(vecID != XXH_SCALAR);
|
|
#endif
|
|
#if !XXH_DISPATCH_AVX512
|
|
assert(vecID != XXH_AVX512);
|
|
#endif
|
|
#if !XXH_DISPATCH_AVX2
|
|
assert(vecID != XXH_AVX2);
|
|
#endif
|
|
XXH_g_dispatch = XXH_kDispatch[vecID];
|
|
XXH_g_dispatch128 = XXH_kDispatch128[vecID];
|
|
}
|
|
|
|
|
|
/* ==== XXH3 public functions ==== */
|
|
|
|
static XXH64_hash_t
|
|
XXH3_hashLong_64b_defaultSecret_selection(const void* input, size_t len,
|
|
XXH64_hash_t seed64, const xxh_u8* secret, size_t secretLen)
|
|
{
|
|
(void)seed64; (void)secret; (void)secretLen;
|
|
if (XXH_DISPATCH_MAYBE_NULL && XXH_g_dispatch.hashLong64_default == NULL)
|
|
XXH_setDispatch();
|
|
return XXH_g_dispatch.hashLong64_default(input, len);
|
|
}
|
|
|
|
XXH64_hash_t XXH3_64bits_dispatch(XXH_NOESCAPE const void* input, size_t len)
|
|
{
|
|
return XXH3_64bits_internal(input, len, 0, XXH3_kSecret, sizeof(XXH3_kSecret), XXH3_hashLong_64b_defaultSecret_selection);
|
|
}
|
|
|
|
static XXH64_hash_t
|
|
XXH3_hashLong_64b_withSeed_selection(const void* input, size_t len,
|
|
XXH64_hash_t seed64, const xxh_u8* secret, size_t secretLen)
|
|
{
|
|
(void)secret; (void)secretLen;
|
|
if (XXH_DISPATCH_MAYBE_NULL && XXH_g_dispatch.hashLong64_seed == NULL)
|
|
XXH_setDispatch();
|
|
return XXH_g_dispatch.hashLong64_seed(input, len, seed64);
|
|
}
|
|
|
|
XXH64_hash_t XXH3_64bits_withSeed_dispatch(XXH_NOESCAPE const void* input, size_t len, XXH64_hash_t seed)
|
|
{
|
|
return XXH3_64bits_internal(input, len, seed, XXH3_kSecret, sizeof(XXH3_kSecret), XXH3_hashLong_64b_withSeed_selection);
|
|
}
|
|
|
|
static XXH64_hash_t
|
|
XXH3_hashLong_64b_withSecret_selection(const void* input, size_t len,
|
|
XXH64_hash_t seed64, const xxh_u8* secret, size_t secretLen)
|
|
{
|
|
(void)seed64;
|
|
if (XXH_DISPATCH_MAYBE_NULL && XXH_g_dispatch.hashLong64_secret == NULL)
|
|
XXH_setDispatch();
|
|
return XXH_g_dispatch.hashLong64_secret(input, len, secret, secretLen);
|
|
}
|
|
|
|
XXH64_hash_t XXH3_64bits_withSecret_dispatch(XXH_NOESCAPE const void* input, size_t len, XXH_NOESCAPE const void* secret, size_t secretLen)
|
|
{
|
|
return XXH3_64bits_internal(input, len, 0, secret, secretLen, XXH3_hashLong_64b_withSecret_selection);
|
|
}
|
|
|
|
XXH_errorcode
|
|
XXH3_64bits_update_dispatch(XXH_NOESCAPE XXH3_state_t* state, XXH_NOESCAPE const void* input, size_t len)
|
|
{
|
|
if (XXH_DISPATCH_MAYBE_NULL && XXH_g_dispatch.update == NULL)
|
|
XXH_setDispatch();
|
|
|
|
return XXH_g_dispatch.update(state, (const xxh_u8*)input, len);
|
|
}
|
|
|
|
|
|
/* ==== XXH128 public functions ==== */
|
|
|
|
static XXH128_hash_t
|
|
XXH3_hashLong_128b_defaultSecret_selection(const void* input, size_t len,
|
|
XXH64_hash_t seed64, const void* secret, size_t secretLen)
|
|
{
|
|
(void)seed64; (void)secret; (void)secretLen;
|
|
if (XXH_DISPATCH_MAYBE_NULL && XXH_g_dispatch128.hashLong128_default == NULL)
|
|
XXH_setDispatch();
|
|
return XXH_g_dispatch128.hashLong128_default(input, len);
|
|
}
|
|
|
|
XXH128_hash_t XXH3_128bits_dispatch(XXH_NOESCAPE const void* input, size_t len)
|
|
{
|
|
return XXH3_128bits_internal(input, len, 0, XXH3_kSecret, sizeof(XXH3_kSecret), XXH3_hashLong_128b_defaultSecret_selection);
|
|
}
|
|
|
|
static XXH128_hash_t
|
|
XXH3_hashLong_128b_withSeed_selection(const void* input, size_t len,
|
|
XXH64_hash_t seed64, const void* secret, size_t secretLen)
|
|
{
|
|
(void)secret; (void)secretLen;
|
|
if (XXH_DISPATCH_MAYBE_NULL && XXH_g_dispatch128.hashLong128_seed == NULL)
|
|
XXH_setDispatch();
|
|
return XXH_g_dispatch128.hashLong128_seed(input, len, seed64);
|
|
}
|
|
|
|
XXH128_hash_t XXH3_128bits_withSeed_dispatch(XXH_NOESCAPE const void* input, size_t len, XXH64_hash_t seed)
|
|
{
|
|
return XXH3_128bits_internal(input, len, seed, XXH3_kSecret, sizeof(XXH3_kSecret), XXH3_hashLong_128b_withSeed_selection);
|
|
}
|
|
|
|
static XXH128_hash_t
|
|
XXH3_hashLong_128b_withSecret_selection(const void* input, size_t len,
|
|
XXH64_hash_t seed64, const void* secret, size_t secretLen)
|
|
{
|
|
(void)seed64;
|
|
if (XXH_DISPATCH_MAYBE_NULL && XXH_g_dispatch128.hashLong128_secret == NULL)
|
|
XXH_setDispatch();
|
|
return XXH_g_dispatch128.hashLong128_secret(input, len, secret, secretLen);
|
|
}
|
|
|
|
XXH128_hash_t XXH3_128bits_withSecret_dispatch(XXH_NOESCAPE const void* input, size_t len, XXH_NOESCAPE const void* secret, size_t secretLen)
|
|
{
|
|
return XXH3_128bits_internal(input, len, 0, secret, secretLen, XXH3_hashLong_128b_withSecret_selection);
|
|
}
|
|
|
|
XXH_errorcode
|
|
XXH3_128bits_update_dispatch(XXH_NOESCAPE XXH3_state_t* state, XXH_NOESCAPE const void* input, size_t len)
|
|
{
|
|
if (XXH_DISPATCH_MAYBE_NULL && XXH_g_dispatch128.update == NULL)
|
|
XXH_setDispatch();
|
|
return XXH_g_dispatch128.update(state, (const xxh_u8*)input, len);
|
|
}
|
|
|
|
#if defined (__cplusplus)
|
|
}
|
|
#endif
|
|
/*! @} */
|
|
|
|
#endif /* __x86_64__ */
|