vbatts/llama.cpp
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

fallback mmvq to mul_mat

Browse source
This commit is contained in:
Meng, Hengyu 2024-08-19 03:50:48 +00:00
parent cfac111e2b
commit 5b6d224695
2 changed files with 2 additions and 1 deletions
Download patch file Download diff file Expand all files Collapse all files

2
ggml/src/ggml-sycl.cpp
View file

@ -3581,7 +3581,7 @@ static void ggml_sycl_mul_mat(ggml_backend_sycl_context & ctx, const ggml_tensor
bool use_mul_mat_vec_q = ggml_is_quantized(src0->type) bool use_mul_mat_vec_q = ggml_is_quantized(src0->type)
&& src1->type == GGML_TYPE_F32 && dst->type == GGML_TYPE_F32 && src1->type == GGML_TYPE_F32 && dst->type == GGML_TYPE_F32
&& src1->ne[1] <= MMVQ_MAX_BATCH_SIZE; && src1->ne[1] <= MMVQ_MAX_BATCH_SIZE && src1->ne[1] > MMVQ_MIN_BATCH_SIZE;
bool use_mul_mat_q = ggml_sycl_supports_mmq(src0->type) bool use_mul_mat_q = ggml_sycl_supports_mmq(src0->type)
&& src1->type == GGML_TYPE_F32 && dst->type == GGML_TYPE_F32; && src1->type == GGML_TYPE_F32 && dst->type == GGML_TYPE_F32;

1
ggml/src/ggml-sycl/common.hpp
View file

@ -130,6 +130,7 @@ typedef sycl::float2 dfloat2;
#endif // GGML_SYCL_F16 #endif // GGML_SYCL_F16
#define MMVQ_MAX_BATCH_SIZE 8 #define MMVQ_MAX_BATCH_SIZE 8
#define MMVQ_MIN_BATCH_SIZE 4
static const int8_t kvalues_iq4nl[16]={-127, -104, -83, -65, -49, -35, -22, -10, 1, 13, 25, 38, 53, 69, 89, 113}; static const int8_t kvalues_iq4nl[16]={-127, -104, -83, -65, -49, -35, -22, -10, 1, 13, 25, 38, 53, 69, 89, 113};