vbatts/llama.cpp
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

llama : fix attention layer count sanity check

Browse source
This commit is contained in:
Georgi Gerganov 2024-04-08 21:17:40 +03:00
parent cecd8d3c98
commit 6804714190
No known key found for this signature in database
GPG key ID: 449E073F9DC10735
1 changed files with 7 additions and 2 deletions
Download patch file Download diff file Expand all files Collapse all files

9
llama.cpp
View file

@ -13468,7 +13468,8 @@ static void llama_model_quantize_internal(const std::string & fname_inp, const s
const std::string name = ggml_get_name(meta);
// TODO: avoid hardcoded tensor names - use the TN_* constants
if (name.find("attn_v.weight") != std::string::npos || name.find("attn_qkv.weight") != std::string::npos) {
if (name.find("attn_v.weight") != std::string::npos ||
name.find("attn_qkv.weight") != std::string::npos) {
++qs.n_attention_wv;
} else if (name == LLM_TN(model.arch)(LLM_TENSOR_OUTPUT, "weight")) {
qs.has_output = true;
@ -13478,7 +13479,11 @@ static void llama_model_quantize_internal(const std::string & fname_inp, const s
qs.n_ffn_down = qs.n_ffn_gate = qs.n_ffn_up = (int)model.hparams.n_layer;
// sanity checks
GGML_ASSERT(qs.n_attention_wv == (int)model.hparams.n_layer && "n_attention_wv != n_layer is unexpected");
//
// - qs.n_attention_wv == 0 for Mamba models
// - qs.n_attention_wv == model.hparams.n_layer for Transformer models
//
GGML_ASSERT(qs.n_attention_wv == 0 || qs.n_attention_wv == (int)model.hparams.n_layer && "n_attention_wv is unexpected");
size_t total_size_org = 0;
size_t total_size_new = 0;