mirror of
https://github.com/jart/cosmopolitan.git
synced 2025-01-31 11:37:35 +00:00
c9152b6f14
This change switches c++ exception handling from sjlj to standard dwarf. It's needed because clang for aarch64 doesn't support sjlj. It turns out that libunwind had a bare-metal configuration that made this easy to do. This change gets the new experimental cosmocc -mclang flag in a state of working so well that it can now be used to build all of llamafile and it goes 3x faster in terms of build latency, without trading away any perf. The int_fast16_t and int_fast32_t types are now always defined as 32-bit in the interest of having more abi consistency between cosmocc -mgcc and -mclang mode.
484 lines
14 KiB
C
484 lines
14 KiB
C
/*===-------------- avxneconvertintrin.h - AVXNECONVERT --------------------===
|
|
*
|
|
* Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
|
|
* See https://llvm.org/LICENSE.txt for license information.
|
|
* SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
|
|
*
|
|
*===-----------------------------------------------------------------------===
|
|
*/
|
|
|
|
#ifndef __IMMINTRIN_H
|
|
#error \
|
|
"Never use <avxneconvertintrin.h> directly; include <immintrin.h> instead."
|
|
#endif // __IMMINTRIN_H
|
|
|
|
#ifdef __SSE2__
|
|
|
|
#ifndef __AVXNECONVERTINTRIN_H
|
|
#define __AVXNECONVERTINTRIN_H
|
|
|
|
/* Define the default attributes for the functions in this file. */
|
|
#define __DEFAULT_FN_ATTRS128 \
|
|
__attribute__((__always_inline__, __nodebug__, __target__("avxneconvert"), \
|
|
__min_vector_width__(128)))
|
|
#define __DEFAULT_FN_ATTRS256 \
|
|
__attribute__((__always_inline__, __nodebug__, __target__("avxneconvert"), \
|
|
__min_vector_width__(256)))
|
|
|
|
/// Convert scalar BF16 (16-bit) floating-point element
|
|
/// stored at memory locations starting at location \a __A to a
|
|
/// single-precision (32-bit) floating-point, broadcast it to packed
|
|
/// single-precision (32-bit) floating-point elements, and store the results in
|
|
/// \a dst.
|
|
///
|
|
/// \headerfile <x86intrin.h>
|
|
///
|
|
/// \code
|
|
/// _mm_bcstnebf16_ps(const void *__A);
|
|
/// \endcode
|
|
///
|
|
/// This intrinsic corresponds to the \c VBCSTNEBF162PS instruction.
|
|
///
|
|
/// \param __A
|
|
/// A pointer to a 16-bit memory location. The address of the memory
|
|
/// location does not have to be aligned.
|
|
/// \returns
|
|
/// A 128-bit vector of [4 x float].
|
|
///
|
|
/// \code{.operation}
|
|
/// b := Convert_BF16_To_FP32(MEM[__A+15:__A])
|
|
/// FOR j := 0 to 3
|
|
/// m := j*32
|
|
/// dst[m+31:m] := b
|
|
/// ENDFOR
|
|
/// dst[MAX:128] := 0
|
|
/// \endcode
|
|
static __inline__ __m128 __DEFAULT_FN_ATTRS128
|
|
_mm_bcstnebf16_ps(const void *__A) {
|
|
return (__m128)__builtin_ia32_vbcstnebf162ps128((const __bf16 *)__A);
|
|
}
|
|
|
|
/// Convert scalar BF16 (16-bit) floating-point element
|
|
/// stored at memory locations starting at location \a __A to a
|
|
/// single-precision (32-bit) floating-point, broadcast it to packed
|
|
/// single-precision (32-bit) floating-point elements, and store the results in
|
|
/// \a dst.
|
|
///
|
|
/// \headerfile <x86intrin.h>
|
|
///
|
|
/// \code
|
|
/// _mm256_bcstnebf16_ps(const void *__A);
|
|
/// \endcode
|
|
///
|
|
/// This intrinsic corresponds to the \c VBCSTNEBF162PS instruction.
|
|
///
|
|
/// \param __A
|
|
/// A pointer to a 16-bit memory location. The address of the memory
|
|
/// location does not have to be aligned.
|
|
/// \returns
|
|
/// A 256-bit vector of [8 x float].
|
|
///
|
|
/// \code{.operation}
|
|
/// b := Convert_BF16_To_FP32(MEM[__A+15:__A])
|
|
/// FOR j := 0 to 7
|
|
/// m := j*32
|
|
/// dst[m+31:m] := b
|
|
/// ENDFOR
|
|
/// dst[MAX:256] := 0
|
|
/// \endcode
|
|
static __inline__ __m256 __DEFAULT_FN_ATTRS256
|
|
_mm256_bcstnebf16_ps(const void *__A) {
|
|
return (__m256)__builtin_ia32_vbcstnebf162ps256((const __bf16 *)__A);
|
|
}
|
|
|
|
/// Convert scalar half-precision (16-bit) floating-point element
|
|
/// stored at memory locations starting at location \a __A to a
|
|
/// single-precision (32-bit) floating-point, broadcast it to packed
|
|
/// single-precision (32-bit) floating-point elements, and store the results in
|
|
/// \a dst.
|
|
///
|
|
/// \headerfile <x86intrin.h>
|
|
///
|
|
/// \code
|
|
/// _mm_bcstnesh_ps(const void *__A);
|
|
/// \endcode
|
|
///
|
|
/// This intrinsic corresponds to the \c VBCSTNESH2PS instruction.
|
|
///
|
|
/// \param __A
|
|
/// A pointer to a 16-bit memory location. The address of the memory
|
|
/// location does not have to be aligned.
|
|
/// \returns
|
|
/// A 128-bit vector of [4 x float].
|
|
///
|
|
/// \code{.operation}
|
|
/// b := Convert_FP16_To_FP32(MEM[__A+15:__A])
|
|
/// FOR j := 0 to 3
|
|
/// m := j*32
|
|
/// dst[m+31:m] := b
|
|
/// ENDFOR
|
|
/// dst[MAX:128] := 0
|
|
/// \endcode
|
|
static __inline__ __m128 __DEFAULT_FN_ATTRS128
|
|
_mm_bcstnesh_ps(const void *__A) {
|
|
return (__m128)__builtin_ia32_vbcstnesh2ps128((const _Float16 *)__A);
|
|
}
|
|
|
|
/// Convert scalar half-precision (16-bit) floating-point element
|
|
/// stored at memory locations starting at location \a __A to a
|
|
/// single-precision (32-bit) floating-point, broadcast it to packed
|
|
/// single-precision (32-bit) floating-point elements, and store the results in
|
|
/// \a dst.
|
|
///
|
|
/// \headerfile <x86intrin.h>
|
|
///
|
|
/// \code
|
|
/// _mm256_bcstnesh_ps(const void *__A);
|
|
/// \endcode
|
|
///
|
|
/// This intrinsic corresponds to the \c VBCSTNESH2PS instruction.
|
|
///
|
|
/// \param __A
|
|
/// A pointer to a 16-bit memory location. The address of the memory
|
|
/// location does not have to be aligned.
|
|
/// \returns
|
|
/// A 256-bit vector of [8 x float].
|
|
///
|
|
/// \code{.operation}
|
|
/// b := Convert_FP16_To_FP32(MEM[__A+15:__A])
|
|
/// FOR j := 0 to 7
|
|
/// m := j*32
|
|
/// dst[m+31:m] := b
|
|
/// ENDFOR
|
|
/// dst[MAX:256] := 0
|
|
/// \endcode
|
|
static __inline__ __m256 __DEFAULT_FN_ATTRS256
|
|
_mm256_bcstnesh_ps(const void *__A) {
|
|
return (__m256)__builtin_ia32_vbcstnesh2ps256((const _Float16 *)__A);
|
|
}
|
|
|
|
/// Convert packed BF16 (16-bit) floating-point even-indexed elements
|
|
/// stored at memory locations starting at location \a __A to packed
|
|
/// single-precision (32-bit) floating-point elements, and store the results in
|
|
/// \a dst.
|
|
///
|
|
/// \headerfile <x86intrin.h>
|
|
///
|
|
/// \code
|
|
/// _mm_cvtneebf16_ps(const __m128bh *__A);
|
|
/// \endcode
|
|
///
|
|
/// This intrinsic corresponds to the \c VCVTNEEBF162PS instruction.
|
|
///
|
|
/// \param __A
|
|
/// A pointer to a 128-bit memory location containing 8 consecutive
|
|
/// BF16 (16-bit) floating-point values.
|
|
/// \returns
|
|
/// A 128-bit vector of [4 x float].
|
|
///
|
|
/// \code{.operation}
|
|
/// FOR j := 0 to 3
|
|
/// k := j*2
|
|
/// i := k*16
|
|
/// m := j*32
|
|
/// dst[m+31:m] := Convert_BF16_To_FP32(MEM[__A+i+15:__A+i])
|
|
/// ENDFOR
|
|
/// dst[MAX:128] := 0
|
|
/// \endcode
|
|
static __inline__ __m128 __DEFAULT_FN_ATTRS128
|
|
_mm_cvtneebf16_ps(const __m128bh *__A) {
|
|
return (__m128)__builtin_ia32_vcvtneebf162ps128((const __v8bf *)__A);
|
|
}
|
|
|
|
/// Convert packed BF16 (16-bit) floating-point even-indexed elements
|
|
/// stored at memory locations starting at location \a __A to packed
|
|
/// single-precision (32-bit) floating-point elements, and store the results in
|
|
/// \a dst.
|
|
///
|
|
/// \headerfile <x86intrin.h>
|
|
///
|
|
/// \code
|
|
/// _mm256_cvtneebf16_ps(const __m256bh *__A);
|
|
/// \endcode
|
|
///
|
|
/// This intrinsic corresponds to the \c VCVTNEEBF162PS instruction.
|
|
///
|
|
/// \param __A
|
|
/// A pointer to a 256-bit memory location containing 16 consecutive
|
|
/// BF16 (16-bit) floating-point values.
|
|
/// \returns
|
|
/// A 256-bit vector of [8 x float].
|
|
///
|
|
/// \code{.operation}
|
|
/// FOR j := 0 to 7
|
|
/// k := j*2
|
|
/// i := k*16
|
|
/// m := j*32
|
|
/// dst[m+31:m] := Convert_BF16_To_FP32(MEM[__A+i+15:__A+i])
|
|
/// ENDFOR
|
|
/// dst[MAX:256] := 0
|
|
/// \endcode
|
|
static __inline__ __m256 __DEFAULT_FN_ATTRS256
|
|
_mm256_cvtneebf16_ps(const __m256bh *__A) {
|
|
return (__m256)__builtin_ia32_vcvtneebf162ps256((const __v16bf *)__A);
|
|
}
|
|
|
|
/// Convert packed half-precision (16-bit) floating-point even-indexed elements
|
|
/// stored at memory locations starting at location \a __A to packed
|
|
/// single-precision (32-bit) floating-point elements, and store the results in
|
|
/// \a dst.
|
|
///
|
|
/// \headerfile <x86intrin.h>
|
|
///
|
|
/// \code
|
|
/// _mm_cvtneeph_ps(const __m128h *__A);
|
|
/// \endcode
|
|
///
|
|
/// This intrinsic corresponds to the \c VCVTNEEPH2PS instruction.
|
|
///
|
|
/// \param __A
|
|
/// A pointer to a 128-bit memory location containing 8 consecutive
|
|
/// half-precision (16-bit) floating-point values.
|
|
/// \returns
|
|
/// A 128-bit vector of [4 x float].
|
|
///
|
|
/// \code{.operation}
|
|
/// FOR j := 0 to 3
|
|
/// k := j*2
|
|
/// i := k*16
|
|
/// m := j*32
|
|
/// dst[m+31:m] := Convert_FP16_To_FP32(MEM[__A+i+15:__A+i])
|
|
/// ENDFOR
|
|
/// dst[MAX:128] := 0
|
|
/// \endcode
|
|
static __inline__ __m128 __DEFAULT_FN_ATTRS128
|
|
_mm_cvtneeph_ps(const __m128h *__A) {
|
|
return (__m128)__builtin_ia32_vcvtneeph2ps128((const __v8hf *)__A);
|
|
}
|
|
|
|
/// Convert packed half-precision (16-bit) floating-point even-indexed elements
|
|
/// stored at memory locations starting at location \a __A to packed
|
|
/// single-precision (32-bit) floating-point elements, and store the results in
|
|
/// \a dst.
|
|
///
|
|
/// \headerfile <x86intrin.h>
|
|
///
|
|
/// \code
|
|
/// _mm256_cvtneeph_ps(const __m256h *__A);
|
|
/// \endcode
|
|
///
|
|
/// This intrinsic corresponds to the \c VCVTNEEPH2PS instruction.
|
|
///
|
|
/// \param __A
|
|
/// A pointer to a 256-bit memory location containing 16 consecutive
|
|
/// half-precision (16-bit) floating-point values.
|
|
/// \returns
|
|
/// A 256-bit vector of [8 x float].
|
|
///
|
|
/// \code{.operation}
|
|
/// FOR j := 0 to 7
|
|
/// k := j*2
|
|
/// i := k*16
|
|
/// m := j*32
|
|
/// dst[m+31:m] := Convert_FP16_To_FP32(MEM[__A+i+15:__A+i])
|
|
/// ENDFOR
|
|
/// dst[MAX:256] := 0
|
|
/// \endcode
|
|
static __inline__ __m256 __DEFAULT_FN_ATTRS256
|
|
_mm256_cvtneeph_ps(const __m256h *__A) {
|
|
return (__m256)__builtin_ia32_vcvtneeph2ps256((const __v16hf *)__A);
|
|
}
|
|
|
|
/// Convert packed BF16 (16-bit) floating-point odd-indexed elements
|
|
/// stored at memory locations starting at location \a __A to packed
|
|
/// single-precision (32-bit) floating-point elements, and store the results in
|
|
/// \a dst.
|
|
///
|
|
/// \headerfile <x86intrin.h>
|
|
///
|
|
/// \code
|
|
/// _mm_cvtneobf16_ps(const __m128bh *__A);
|
|
/// \endcode
|
|
///
|
|
/// This intrinsic corresponds to the \c VCVTNEOBF162PS instruction.
|
|
///
|
|
/// \param __A
|
|
/// A pointer to a 128-bit memory location containing 8 consecutive
|
|
/// BF16 (16-bit) floating-point values.
|
|
/// \returns
|
|
/// A 128-bit vector of [4 x float].
|
|
///
|
|
/// \code{.operation}
|
|
/// FOR j := 0 to 3
|
|
/// k := j*2+1
|
|
/// i := k*16
|
|
/// m := j*32
|
|
/// dst[m+31:m] := Convert_BF16_To_FP32(MEM[__A+i+15:__A+i])
|
|
/// ENDFOR
|
|
/// dst[MAX:128] := 0
|
|
/// \endcode
|
|
static __inline__ __m128 __DEFAULT_FN_ATTRS128
|
|
_mm_cvtneobf16_ps(const __m128bh *__A) {
|
|
return (__m128)__builtin_ia32_vcvtneobf162ps128((const __v8bf *)__A);
|
|
}
|
|
|
|
/// Convert packed BF16 (16-bit) floating-point odd-indexed elements
|
|
/// stored at memory locations starting at location \a __A to packed
|
|
/// single-precision (32-bit) floating-point elements, and store the results in
|
|
/// \a dst.
|
|
///
|
|
/// \headerfile <x86intrin.h>
|
|
///
|
|
/// \code
|
|
/// _mm256_cvtneobf16_ps(const __m256bh *__A);
|
|
/// \endcode
|
|
///
|
|
/// This intrinsic corresponds to the \c VCVTNEOBF162PS instruction.
|
|
///
|
|
/// \param __A
|
|
/// A pointer to a 256-bit memory location containing 16 consecutive
|
|
/// BF16 (16-bit) floating-point values.
|
|
/// \returns
|
|
/// A 256-bit vector of [8 x float].
|
|
///
|
|
/// \code{.operation}
|
|
/// FOR j := 0 to 7
|
|
/// k := j*2+1
|
|
/// i := k*16
|
|
/// m := j*32
|
|
/// dst[m+31:m] := Convert_BF16_To_FP32(MEM[__A+i+15:__A+i])
|
|
/// ENDFOR
|
|
/// dst[MAX:256] := 0
|
|
/// \endcode
|
|
static __inline__ __m256 __DEFAULT_FN_ATTRS256
|
|
_mm256_cvtneobf16_ps(const __m256bh *__A) {
|
|
return (__m256)__builtin_ia32_vcvtneobf162ps256((const __v16bf *)__A);
|
|
}
|
|
|
|
/// Convert packed half-precision (16-bit) floating-point odd-indexed elements
|
|
/// stored at memory locations starting at location \a __A to packed
|
|
/// single-precision (32-bit) floating-point elements, and store the results in
|
|
/// \a dst.
|
|
///
|
|
/// \headerfile <x86intrin.h>
|
|
///
|
|
/// \code
|
|
/// _mm_cvtneoph_ps(const __m128h *__A);
|
|
/// \endcode
|
|
///
|
|
/// This intrinsic corresponds to the \c VCVTNEOPH2PS instruction.
|
|
///
|
|
/// \param __A
|
|
/// A pointer to a 128-bit memory location containing 8 consecutive
|
|
/// half-precision (16-bit) floating-point values.
|
|
/// \returns
|
|
/// A 128-bit vector of [4 x float].
|
|
///
|
|
/// \code{.operation}
|
|
/// FOR j := 0 to 3
|
|
/// k := j*2+1
|
|
/// i := k*16
|
|
/// m := j*32
|
|
/// dst[m+31:m] := Convert_FP16_To_FP32(MEM[__A+i+15:__A+i])
|
|
/// ENDFOR
|
|
/// dst[MAX:128] := 0
|
|
/// \endcode
|
|
static __inline__ __m128 __DEFAULT_FN_ATTRS128
|
|
_mm_cvtneoph_ps(const __m128h *__A) {
|
|
return (__m128)__builtin_ia32_vcvtneoph2ps128((const __v8hf *)__A);
|
|
}
|
|
|
|
/// Convert packed half-precision (16-bit) floating-point odd-indexed elements
|
|
/// stored at memory locations starting at location \a __A to packed
|
|
/// single-precision (32-bit) floating-point elements, and store the results in
|
|
/// \a dst.
|
|
///
|
|
/// \headerfile <x86intrin.h>
|
|
///
|
|
/// \code
|
|
/// _mm256_cvtneoph_ps(const __m256h *__A);
|
|
/// \endcode
|
|
///
|
|
/// This intrinsic corresponds to the \c VCVTNEOPH2PS instruction.
|
|
///
|
|
/// \param __A
|
|
/// A pointer to a 256-bit memory location containing 16 consecutive
|
|
/// half-precision (16-bit) floating-point values.
|
|
/// \returns
|
|
/// A 256-bit vector of [8 x float].
|
|
///
|
|
/// \code{.operation}
|
|
/// FOR j := 0 to 7
|
|
/// k := j*2+1
|
|
/// i := k*16
|
|
/// m := j*32
|
|
/// dst[m+31:m] := Convert_FP16_To_FP32(MEM[__A+i+15:__A+i])
|
|
/// ENDFOR
|
|
/// dst[MAX:256] := 0
|
|
/// \endcode
|
|
static __inline__ __m256 __DEFAULT_FN_ATTRS256
|
|
_mm256_cvtneoph_ps(const __m256h *__A) {
|
|
return (__m256)__builtin_ia32_vcvtneoph2ps256((const __v16hf *)__A);
|
|
}
|
|
|
|
/// Convert packed single-precision (32-bit) floating-point elements in \a __A
|
|
/// to packed BF16 (16-bit) floating-point elements, and store the results in \a
|
|
/// dst.
|
|
///
|
|
/// \headerfile <x86intrin.h>
|
|
///
|
|
/// \code
|
|
/// _mm_cvtneps_avx_pbh(__m128 __A);
|
|
/// \endcode
|
|
///
|
|
/// This intrinsic corresponds to the \c VCVTNEPS2BF16 instruction.
|
|
///
|
|
/// \param __A
|
|
/// A 128-bit vector of [4 x float].
|
|
/// \returns
|
|
/// A 128-bit vector of [8 x bfloat].
|
|
///
|
|
/// \code{.operation}
|
|
/// FOR j := 0 to 3
|
|
/// dst.word[j] := Convert_FP32_To_BF16(__A.fp32[j])
|
|
/// ENDFOR
|
|
/// dst[MAX:128] := 0
|
|
/// \endcode
|
|
static __inline__ __m128bh __DEFAULT_FN_ATTRS128
|
|
_mm_cvtneps_avx_pbh(__m128 __A) {
|
|
return (__m128bh)__builtin_ia32_vcvtneps2bf16128((__v4sf)__A);
|
|
}
|
|
|
|
/// Convert packed single-precision (32-bit) floating-point elements in \a __A
|
|
/// to packed BF16 (16-bit) floating-point elements, and store the results in \a
|
|
/// dst.
|
|
///
|
|
/// \headerfile <x86intrin.h>
|
|
///
|
|
/// \code
|
|
/// _mm256_cvtneps_avx_pbh(__m256 __A);
|
|
/// \endcode
|
|
///
|
|
/// This intrinsic corresponds to the \c VCVTNEPS2BF16 instruction.
|
|
///
|
|
/// \param __A
|
|
/// A 256-bit vector of [8 x float].
|
|
/// \returns
|
|
/// A 128-bit vector of [8 x bfloat].
|
|
///
|
|
/// \code{.operation}
|
|
/// FOR j := 0 to 7
|
|
/// dst.word[j] := Convert_FP32_To_BF16(a.fp32[j])
|
|
/// ENDFOR
|
|
/// dst[MAX:128] := 0
|
|
/// \endcode
|
|
static __inline__ __m128bh __DEFAULT_FN_ATTRS256
|
|
_mm256_cvtneps_avx_pbh(__m256 __A) {
|
|
return (__m128bh)__builtin_ia32_vcvtneps2bf16256((__v8sf)__A);
|
|
}
|
|
|
|
#undef __DEFAULT_FN_ATTRS128
|
|
#undef __DEFAULT_FN_ATTRS256
|
|
|
|
#endif // __AVXNECONVERTINTRIN_H
|
|
#endif // __SSE2__
|