vbatts/llama.cpp
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

wip : 8 rows per simd group

Browse source
This commit is contained in:
Georgi Gerganov 2024-01-21 12:23:22 +02:00
parent b97325800a
commit 8cde449b8b
No known key found for this signature in database
GPG key ID: 449E073F9DC10735
2 changed files with 140 additions and 45 deletions
Download patch file Download diff file Expand all files Collapse all files

10
ggml-metal.m
View file

@ -2252,14 +2252,14 @@ static bool ggml_metal_graph_compute(
[encoder setBytes:&ne3 length:sizeof( int64_t) atIndex:26];
[encoder setBytes:&scale length:sizeof( float) atIndex:27];
const int nwarps = 1;
const int64_t nwarps = 2;
const size_t shalf = sizeof(float)/2;
const size_t smem = nwarps*(2*8*nwarps*ne00 + 128)*(sizeof(float)/2);
GGML_ASSERT(2*32*nwarps*ne00*shalf <= ctx->device.maxThreadgroupMemoryLength);
[encoder setThreadgroupMemoryLength:2*32*nwarps*ne00*shalf atIndex:0];
GGML_ASSERT(smem <= ctx->device.maxThreadgroupMemoryLength);
[encoder setThreadgroupMemoryLength:smem atIndex:0];
[encoder dispatchThreadgroups:MTLSizeMake((ne01 + 31)/32, ne02, ne03) threadsPerThreadgroup:MTLSizeMake(32, 1, 1)];
[encoder dispatchThreadgroups:MTLSizeMake(ne01, (ne02 + 8*nwarps - 1)/(8*nwarps), ne03) threadsPerThreadgroup:MTLSizeMake(32*nwarps, 1, 1)];
} break;
case GGML_OP_DUP:
case GGML_OP_CPY: