vbatts/llama.cpp
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

better comments, and fix some small errors.

Browse source
This commit is contained in:
Julia Longtin 2024-03-24 12:17:06 +00:00
parent 0c01d07835
commit 98c9b6972a
1 changed files with 14 additions and 10 deletions
Download patch file Download diff file Expand all files Collapse all files

24
ggml-phi-knc-dot_q5_K_q8_K.c
View file

@ -30,6 +30,7 @@ typedef int32_t int32x16_t __attribute__((vector_size (128)));
/* A forward declaration, to keep GCC happy. */ /* A forward declaration, to keep GCC happy. */
void ggml_vec_dot_q5_K_q8_K(int n, float * restrict s, size_t bs, const void * restrict vx, size_t bx, const void * restrict vy, size_t by, int nrc); void ggml_vec_dot_q5_K_q8_K(int n, float * restrict s, size_t bs, const void * restrict vx, size_t bx, const void * restrict vy, size_t by, int nrc);
/* clear a vector of 8 floats. */
inline static void GGML_F32x8_VEC_ZERO(float32x8_t *target) inline static void GGML_F32x8_VEC_ZERO(float32x8_t *target)
{ {
uint8_t zero[4] __attribute__((aligned(64))) = {0,0,0,0}; uint8_t zero[4] __attribute__((aligned(64))) = {0,0,0,0};
@ -45,6 +46,7 @@ inline static void GGML_F32x8_VEC_ZERO(float32x8_t *target)
: "zmm8", "k1", "memory"); : "zmm8", "k1", "memory");
} }
/* clear a vector of 8 int32_ts. */
inline static void GGML_I32x8_VEC_ZERO(int32x8_t *target) inline static void GGML_I32x8_VEC_ZERO(int32x8_t *target)
{ {
uint8_t zero[4] __attribute__((aligned(64))) = {0,0,0,0}; uint8_t zero[4] __attribute__((aligned(64))) = {0,0,0,0};
@ -60,7 +62,8 @@ inline static void GGML_I32x8_VEC_ZERO(int32x8_t *target)
: "zmm8", "k1", "memory"); : "zmm8", "k1", "memory");
} }
inline static void GGML_I32x16_VEC_ZERO(int32x8_t *target) /* clear a vector of 16 int32_ts. */
inline static void GGML_I32x16_VEC_ZERO(int32x16_t *target)
{ {
uint8_t zero[4] __attribute__((aligned(64))) = {0,0,0,0}; uint8_t zero[4] __attribute__((aligned(64))) = {0,0,0,0};
@ -73,7 +76,7 @@ inline static void GGML_I32x16_VEC_ZERO(int32x8_t *target)
: "zmm8", "k1", "memory"); : "zmm8", "k1", "memory");
} }
// perform an eight wide Fused Multiply Add of an I16x8 times scalar S into I32x8. // perform a Fused Multiply Add of an I16x8 times scalar S into I32x8.
inline static void GGML_I16x8_S_FMA_I32x8 (int16x8_t *src, int32_t scale, int32x8_t *dest) inline static void GGML_I16x8_S_FMA_I32x8 (int16x8_t *src, int32_t scale, int32x8_t *dest)
{ {
uint8_t zero[4] __attribute__((aligned(64))) = {0,0,0,0}; uint8_t zero[4] __attribute__((aligned(64))) = {0,0,0,0};
@ -95,8 +98,8 @@ inline static void GGML_I16x8_S_FMA_I32x8 (int16x8_t *src, int32_t scale, int32x
: "zmm0", "zmm1", "zmm2", "k1", "memory"); : "zmm0", "zmm1", "zmm2", "k1", "memory");
} }
// perform an eight wide Fused Multiply Add of an I16x16 times scalar S into I32x16. // perform a Fused Multiply Add of an I16x16 times scalar S into I32x16.
inline static void GGML_I16x16_S_FMA_I32x16 (int16x8_t *src, int32_t scale, int32x8_t *dest) inline static void GGML_I16x16_S_FMA_I32x16 (int16x16_t *src, int32_t scale, int32x16_t *dest)
{ {
int32_t scaleVec[4] = {scale, scale, scale, scale}; int32_t scaleVec[4] = {scale, scale, scale, scale};
@ -131,8 +134,8 @@ void ggml_vec_dot_q5_K_q8_K(int n, float * restrict s, size_t bs, const void * r
const uint8_t * mins = (const uint8_t*)&utmp[2]; const uint8_t * mins = (const uint8_t*)&utmp[2];
int8_t aux8[QK_K]; int8_t aux8[QK_K];
int16x16_t aux16 __attribute__((aligned(128))); float32x16_t sums __attribute__((aligned(128)));
float32x16_t sums __attribute__((aligned(64))); int16x16_t aux16 __attribute__((aligned(64)));
int32x16_t aux32 __attribute__((aligned(128))); int32x16_t aux32 __attribute__((aligned(128)));
GGML_F32x16_VEC_ZERO(&sums); GGML_F32x16_VEC_ZERO(&sums);
@ -143,8 +146,6 @@ void ggml_vec_dot_q5_K_q8_K(int n, float * restrict s, size_t bs, const void * r
const uint8_t * restrict hm = x[i].qh; const uint8_t * restrict hm = x[i].qh;
const int8_t * restrict q8 = y[i].qs; const int8_t * restrict q8 = y[i].qs;
GGML_I32x16_VEC_ZERO(&aux32);
int8_t * restrict a = aux8; int8_t * restrict a = aux8;
uint8_t m = 1; uint8_t m = 1;
for (int j = 0; j < QK_K/64; ++j) { for (int j = 0; j < QK_K/64; ++j) {
@ -164,17 +165,20 @@ void ggml_vec_dot_q5_K_q8_K(int n, float * restrict s, size_t bs, const void * r
utmp[0] &= kmask1; utmp[0] &= kmask1;
int sumi = 0; int sumi = 0;
GGML_I32x16_VEC_ZERO(&aux32);
for (int j = 0; j < QK_K/16; ++j) sumi += y[i].bsums[j] * mins[j/2]; for (int j = 0; j < QK_K/16; ++j) sumi += y[i].bsums[j] * mins[j/2];
a = aux8; a = aux8;
int is = 0; int is = 0;
for (int j = 0; j < QK_K/32; ++j) { for (int j = 0; j < QK_K/32; ++j) {
int32_t scale = scales[is++]; int32_t scale = scales[is++];
for (int l = 0; l < 16; ++l) ((int16_t *)&aux16)[l] = q8[l] * a[l]; for (int l = 0; l < 16; ++l) ((int16_t *)&aux16)[l] = q8[l] * a[l];
GGML_I16x8_S_FMA_I32x16 (&aux16, scale, &aux32); GGML_I16x16_S_FMA_I32x16 (&aux16, scale, &aux32);
q8 += 16; a += 16; q8 += 16; a += 16;
/* FIXME: while comparing FMA output to normal output, the original had an error. hunt it down. */ /* FIXME: while comparing FMA output to normal output, the original had an error. hunt it down. */
for (int l = 0; l < 16; ++l) ((int16_t *)&aux16)[l] = q8[l] * a[l]; for (int l = 0; l < 16; ++l) ((int16_t *)&aux16)[l] = q8[l] * a[l];
GGML_I16x8_S_FMA_I32x16 (&aux16, scale, &aux32); GGML_I16x16_S_FMA_I32x16 (&aux16, scale, &aux32);
q8 += 16; a += 16; q8 += 16; a += 16;
} }
const float d = GGML_FP16_TO_FP32(x[i].d) * y[i].d; const float d = GGML_FP16_TO_FP32(x[i].d) * y[i].d;