vbatts/llama.cpp
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Merge branch 'kquant_vocab_fix' into concedo_experimental

Browse source
This commit is contained in:
Concedo 2023-07-10 23:28:48 +08:00
commit 11ebfea8c0
1 changed files with 10 additions and 4 deletions
Download patch file Download diff file Expand all files Collapse all files

14
llama.cpp
View file

@ -2431,7 +2431,6 @@ static void llama_model_quantize_internal(const std::string & fname_inp, const s
int ny = tensor.ne.at(1); int ny = tensor.ne.at(1);
if (nx % QK_K != 0 || ny % QK_K != 0) { if (nx % QK_K != 0 || ny % QK_K != 0) {
fprintf(stderr, "\n\nTensor sizes %d x %d are not divisible by %d, required for k-quants.\n",nx,ny,QK_K); fprintf(stderr, "\n\nTensor sizes %d x %d are not divisible by %d, required for k-quants.\n",nx,ny,QK_K);
fprintf(stderr, "Q8_0 will be used for this tensor instead.\n");
convert_incompatible_tensor = true; convert_incompatible_tensor = true;
} }
} }
@ -2460,9 +2459,16 @@ static void llama_model_quantize_internal(const std::string & fname_inp, const s
if (ftype == LLAMA_FTYPE_MOSTLY_Q3_K_M || ftype == LLAMA_FTYPE_MOSTLY_Q2_K) new_type = GGML_TYPE_Q4_K; if (ftype == LLAMA_FTYPE_MOSTLY_Q3_K_M || ftype == LLAMA_FTYPE_MOSTLY_Q2_K) new_type = GGML_TYPE_Q4_K;
else if (ftype == LLAMA_FTYPE_MOSTLY_Q3_K_L) new_type = GGML_TYPE_Q5_K; else if (ftype == LLAMA_FTYPE_MOSTLY_Q3_K_L) new_type = GGML_TYPE_Q5_K;
} }
if(convert_incompatible_tensor) if (convert_incompatible_tensor) {
{ if (tensor.name == "output.weight") {
new_type = GGML_TYPE_Q8_0; //fall back to Q8_0 instead of just failing. new_type = GGML_TYPE_F16; //fall back to F16 instead of just failing.
fprintf(stderr, "F16 will be used for this tensor instead.\n");
} else if (tensor.name == "tok_embeddings.weight") {
new_type = GGML_TYPE_Q4_0; //fall back to Q4_0 instead of just failing.
fprintf(stderr, "Q4_0 will be used for this tensor instead.\n");
} else {
throw std::runtime_error("Unsupported tensor size encountered\n");
}
} }
#endif #endif