vbatts/llama.cpp
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

metal : require ne00 >= 128 for mat-mat kernels

Browse source 
ggml-ci
This commit is contained in:
Georgi Gerganov 2024-03-20 16:36:17 +02:00
parent 91f8ad167d
commit 712b5d6344
No known key found for this signature in database
GPG key ID: BF970631944C16B7
3 changed files with 10 additions and 3 deletions
Download patch file Download diff file Expand all files Collapse all files

2
examples/batched/batched.cpp
View file

@ -78,7 +78,7 @@ int main(int argc, char ** argv) {
llama_context_params ctx_params = llama_context_default_params(); llama_context_params ctx_params = llama_context_default_params();
ctx_params.seed = 1234; ctx_params.seed = 1234;
ctx_params.n_ctx = n_kv_req; ctx_params.n_ctx = n_kv_req;
ctx_params.n_batch = std::max(n_len, n_parallel); ctx_params.n_batch = std::max(n_len, n_parallel);
ctx_params.n_seq_max = n_parallel; ctx_params.n_seq_max = n_parallel;
ctx_params.n_threads = params.n_threads; ctx_params.n_threads = params.n_threads;

4
ggml-metal.m
View file

@ -1383,7 +1383,7 @@ static enum ggml_status ggml_metal_graph_compute(
!ggml_is_transposed(src0) && !ggml_is_transposed(src0) &&
!ggml_is_transposed(src1) && !ggml_is_transposed(src1) &&
src1t == GGML_TYPE_F32 && src1t == GGML_TYPE_F32 &&
ne00 % 32 == 0 && ne00 >= 64 && ne00 % 32 == 0 && ne00 >= 128 &&
(ne11 > ne11_mm_min || (ggml_is_quantized(src0t) && ne12 > 1))) { (ne11 > ne11_mm_min || (ggml_is_quantized(src0t) && ne12 > 1))) {
//printf("matrix: ne00 = %6d, ne01 = %6d, ne02 = %6d, ne11 = %6d, ne12 = %6d\n", ne00, ne01, ne02, ne11, ne12); //printf("matrix: ne00 = %6d, ne01 = %6d, ne02 = %6d, ne11 = %6d, ne12 = %6d\n", ne00, ne01, ne02, ne11, ne12);
@ -1698,7 +1698,7 @@ static enum ggml_status ggml_metal_graph_compute(
// indirect matrix multiplication // indirect matrix multiplication
// !!! // !!!
if ([ctx->device supportsFamily:MTLGPUFamilyApple7] && if ([ctx->device supportsFamily:MTLGPUFamilyApple7] &&
ne20 % 32 == 0 && ne20 >= 64 && ne20 % 32 == 0 && ne20 >= 128 &&
ne11 > ne11_mm_min) { ne11 > ne11_mm_min) {
id<MTLComputePipelineState> pipeline = nil; id<MTLComputePipelineState> pipeline = nil;

7
tests/test-backend-ops.cpp
View file

@ -2091,6 +2091,13 @@ static bool test_backend(ggml_backend_t backend, test_mode mode, const char * op
} }
} }
test_cases.emplace_back(new test_mul_mat(GGML_TYPE_F16, GGML_TYPE_F32, 64, 2, 128, { 8, 1}, {1, 1}));
test_cases.emplace_back(new test_mul_mat(GGML_TYPE_F16, GGML_TYPE_F32, 83, 2, 128, { 8, 1}, {4, 1}));
test_cases.emplace_back(new test_mul_mat(GGML_TYPE_F16, GGML_TYPE_F32, 64, 2, 64, { 8, 1}, {4, 1}));
test_cases.emplace_back(new test_mul_mat(GGML_TYPE_F16, GGML_TYPE_F32, 83, 2, 64, { 8, 1}, {4, 1}));
test_cases.emplace_back(new test_mul_mat(GGML_TYPE_F16, GGML_TYPE_F32, 64, 45, 128, { 8, 1}, {4, 1}));
test_cases.emplace_back(new test_mul_mat(GGML_TYPE_F16, GGML_TYPE_F32, 128, 45, 64, { 8, 1}, {4, 1}));
for (ggml_type type_a : all_types) { for (ggml_type type_a : all_types) {
for (ggml_type type_b : {GGML_TYPE_F32 /*, GGML_TYPE_F16 */}) { for (ggml_type type_b : {GGML_TYPE_F32 /*, GGML_TYPE_F16 */}) {
for (int n_mats : {2, 4, 8}) { for (int n_mats : {2, 4, 8}) {