From 185416884131b7f7330405e0a730197f5ccd3284 Mon Sep 17 00:00:00 2001
From: Concedo <39025047+LostRuins@users.noreply.github.com>
Date: Sat, 8 Jul 2023 20:31:49 +0800
Subject: [PATCH] This allows LLAMA models that were previously incompatible
 with K quants to function mostly as normal. This happens when a model has a
 vocab != 32000, e.g 32001 which means it's not divisible by 256 or 64. Since
 the problematic dimensions only apply for `tok_embeddings.weight` and
 `output.weight` (dimentions 4096 x n_vocab), we can simply quantize these
 layers to Q8_0 whereas the majority of the hidden layers are still K-quanted
 since they have compatible dimensions.

---
 llama.cpp | 12 ++++++++----
 1 file changed, 8 insertions(+), 4 deletions(-)

diff --git a/llama.cpp b/llama.cpp
index ee6ec0920..be48a4e18 100644
--- a/llama.cpp
+++ b/llama.cpp
@@ -2428,15 +2428,15 @@ static void llama_model_quantize_internal(const std::string & fname_inp, const s
         } else {
             new_type = quantized_type;
 #ifdef GGML_USE_K_QUANTS
+            bool convert_incompatible_tensor = false;
             if (quantized_type == GGML_TYPE_Q2_K || quantized_type == GGML_TYPE_Q3_K || quantized_type == GGML_TYPE_Q4_K ||
                 quantized_type == GGML_TYPE_Q5_K || quantized_type == GGML_TYPE_Q6_K) {
                 int nx = tensor.ne.at(0);
                 int ny = tensor.ne.at(1);
                 if (nx % QK_K != 0 || ny % QK_K != 0) {
-                    fprintf(stderr, "\n\n========================= Tensor sizes %d x %d are not divisible by %d\n",nx,ny,QK_K);
-                    fprintf(stderr, "This is required to be able to use k-quants for now!\n");
-                    fprintf(stderr, "========================================================================================\n\n");
-                    throw std::runtime_error("Unsupported tensor size encountered\n");
+                    fprintf(stderr, "\n\nTensor sizes %d x %d are not divisible by %d, required for k-quants.\n",nx,ny,QK_K);
+                    fprintf(stderr, "Q8_0 will be used for this tensor instead.\n");
+                    convert_incompatible_tensor = true;
                 }
             }
             if (tensor.name == "output.weight") {
@@ -2464,6 +2464,10 @@ static void llama_model_quantize_internal(const std::string & fname_inp, const s
                 if      (ftype == LLAMA_FTYPE_MOSTLY_Q3_K_M || ftype == LLAMA_FTYPE_MOSTLY_Q2_K) new_type = GGML_TYPE_Q4_K;
                 else if (ftype == LLAMA_FTYPE_MOSTLY_Q3_K_L) new_type = GGML_TYPE_Q5_K;
             }
+            if(convert_incompatible_tensor)
+            {
+                new_type = GGML_TYPE_Q8_0; //fall back to Q8_0 instead of just failing.
+            }
 #endif
 
             float * f32_data;