vbatts/llama.cpp
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

wip

Browse source
This commit is contained in:
Georgi Gerganov 2024-01-24 15:45:04 +02:00
parent af3eda9c77
commit 6ccbd1777a
No known key found for this signature in database
GPG key ID: BF970631944C16B7
2 changed files with 117 additions and 128 deletions
Download patch file Download diff file Expand all files Collapse all files

8
ggml-metal.m
View file

@ -2253,13 +2253,15 @@ static bool ggml_metal_graph_compute(
[encoder setBytes:&ne3 length:sizeof( int64_t) atIndex:26];
[encoder setBytes:&scale length:sizeof( float) atIndex:27];
const int64_t nsg = 2; // simdgroups per threadgroup (a.k.a. warps)
const int64_t nhptg = 4; // heads per threadgroup !! sync with kernel template arguments !!
const int64_t nsg = 16; // simdgroups per threadgroup (a.k.a. warps)
const int64_t nhptg = 2; // heads per threadgroup !! sync with kernel template arguments !!
const int64_t nqptg = 2; // queries per threadgroup !! sync with kernel template arguments !!
const int64_t ncpsg = 8;
//const size_t smem = nqptg*(nhptg*ne00 + nsg*(nhptg*ne00 + 256))*(sizeof(float)/2);
const size_t smem = nqptg*(nhptg*ne00 + nsg*(256))*(sizeof(float)/2);
const size_t smem = nqptg*(nhptg*ne00 + nsg*(32*ncpsg))*(sizeof(float)/2);
//printf("smem: %zu, max: %zu\n", smem, ctx->device.maxThreadgroupMemoryLength);
GGML_ASSERT(smem <= ctx->device.maxThreadgroupMemoryLength);
[encoder setThreadgroupMemoryLength:smem atIndex:0];