vbatts/llama.cpp
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

llama : pad n_ctx by 32

Browse source 
ggml-ci
This commit is contained in:
Georgi Gerganov 2024-03-21 14:55:15 +02:00
parent 712b5d6344
commit 1d6112bace
No known key found for this signature in database
GPG key ID: BF970631944C16B7
4 changed files with 8 additions and 3 deletions
Download patch file Download diff file Expand all files Collapse all files

2
examples/batched/batched.cpp
View file

@ -48,6 +48,8 @@ int main(int argc, char ** argv) {
params.prompt = "Hello my name is";
}
process_escapes(params.prompt);
// init LLM
llama_backend_init();

4
ggml-metal.m
View file

@ -1383,7 +1383,7 @@ static enum ggml_status ggml_metal_graph_compute(
!ggml_is_transposed(src0) &&
!ggml_is_transposed(src1) &&
src1t == GGML_TYPE_F32 &&
ne00 % 32 == 0 && ne00 >= 128 &&
ne00 % 32 == 0 && ne00 >= 64 &&
(ne11 > ne11_mm_min || (ggml_is_quantized(src0t) && ne12 > 1))) {
//printf("matrix: ne00 = %6d, ne01 = %6d, ne02 = %6d, ne11 = %6d, ne12 = %6d\n", ne00, ne01, ne02, ne11, ne12);
@ -1698,7 +1698,7 @@ static enum ggml_status ggml_metal_graph_compute(
// indirect matrix multiplication
// !!!
if ([ctx->device supportsFamily:MTLGPUFamilyApple7] &&
ne20 % 32 == 0 && ne20 >= 128 &&
ne20 % 32 == 0 && ne20 >= 64 &&
ne11 > ne11_mm_min) {
id<MTLComputePipelineState> pipeline = nil;

3
llama.cpp
View file

@ -13044,6 +13044,9 @@ struct llama_context * llama_new_context_with_model(
cparams.rope_freq_base = params.rope_freq_base == 0.0f ? hparams.rope_freq_base_train : params.rope_freq_base;
cparams.rope_freq_scale = params.rope_freq_scale == 0.0f ? hparams.rope_freq_scale_train : params.rope_freq_scale;
// this is necessary due to kv_self.n being padded later during inference
cparams.n_ctx = GGML_PAD(cparams.n_ctx, 32);
// with causal attention, the batch size is limited by the context size
cparams.n_batch = hparams.causal_attn ? std::min(cparams.n_ctx, params.n_batch) : params.n_batch;
cparams.n_ubatch = std::min(cparams.n_batch, params.n_ubatch == 0 ? params.n_batch : params.n_ubatch);