vbatts/llama.cpp
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

llama : do not quantize pos embd and token type tensors

Browse source
This commit is contained in:
Georgi Gerganov 2024-02-13 11:17:05 +02:00
parent 895407f31b
commit ce730ad7e3
No known key found for this signature in database
GPG key ID: 449E073F9DC10735
1 changed files with 5 additions and 1 deletions
Download patch file Download diff file Expand all files Collapse all files

6
llama.cpp
View file

@ -10444,7 +10444,11 @@ static void llama_model_quantize_internal(const std::string & fname_inp, const s
quantize &= !params->only_copy; quantize &= !params->only_copy;
// do not quantize expert gating tensors // do not quantize expert gating tensors
quantize &= name.find("ffn_gate_inp.weight") == std::string::npos; quantize &= name != LLM_TN(model.arch)(LLM_TENSOR_FFN_GATE_INP, "weight");
// do not quantize positional embeddings and token types (BERT)
quantize &= name != LLM_TN(model.arch)(LLM_TENSOR_POS_EMBD, "weight");
quantize &= name != LLM_TN(model.arch)(LLM_TENSOR_TOKEN_TYPES, "weight");
enum ggml_type new_type; enum ggml_type new_type;
void * new_data; void * new_data;