vbatts/llama.cpp
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

gemma : use Q8_0 for the token_embd.weight tensor

Browse source
This commit is contained in:
Georgi Gerganov 2024-02-21 23:23:17 +02:00
parent 89febfed93
commit f181e601a1
No known key found for this signature in database
GPG key ID: 449E073F9DC10735
1 changed files with 3 additions and 0 deletions
Download patch file Download diff file Expand all files Collapse all files

3
llama.cpp
View file

@ -10499,6 +10499,9 @@ static ggml_type get_k_quant_type(quantize_state_internal & qs, ggml_type new_ty
else if (ftype == LLAMA_FTYPE_MOSTLY_IQ3_XXS) {
new_type = GGML_TYPE_Q4_K;
}
else if (arch == LLM_ARCH_GEMMA) {
new_type = GGML_TYPE_Q8_0;
}
} else if (ftype == LLAMA_FTYPE_MOSTLY_IQ2_XXS || ftype == LLAMA_FTYPE_MOSTLY_IQ2_XS || ftype == LLAMA_FTYPE_MOSTLY_IQ1_S) {
if (name.find("attn_v.weight") != std::string::npos) {
if (qs.model.hparams.n_gqa() >= 4 || qs.model.hparams.n_expert >= 4) new_type = GGML_TYPE_Q4_K;