From fd9a2fdfe2dd7a54546be2c88b196e193ae7920e Mon Sep 17 00:00:00 2001
From: Concedo <39025047+LostRuins@users.noreply.github.com>
Date: Mon, 10 Jul 2023 23:22:45 +0800
Subject: [PATCH] As an alternative, to avoid failing on Metal due to lack of
 Q8_0 support, instead quantize tok_embeddings.weight to Q4_0 and retain
 output.weight as F16. This results in a net gain of about 55mb for a 7B model
 compared to previous approach, but should minimize adverse impact to model
 quality.

---
 llama.cpp | 11 +++++++++--
 1 file changed, 9 insertions(+), 2 deletions(-)

diff --git a/llama.cpp b/llama.cpp
index 4ab291793..d770a7c8f 100644
--- a/llama.cpp
+++ b/llama.cpp
@@ -2435,7 +2435,6 @@ static void llama_model_quantize_internal(const std::string & fname_inp, const s
                 int ny = tensor.ne.at(1);
                 if (nx % QK_K != 0 || ny % QK_K != 0) {
                     fprintf(stderr, "\n\nTensor sizes %d x %d are not divisible by %d, required for k-quants.\n",nx,ny,QK_K);
-                    fprintf(stderr, "Q8_0 will be used for this tensor instead.\n");
                     convert_incompatible_tensor = true;
                 }
             }
@@ -2465,7 +2464,15 @@ static void llama_model_quantize_internal(const std::string & fname_inp, const s
                 else if (ftype == LLAMA_FTYPE_MOSTLY_Q3_K_L) new_type = GGML_TYPE_Q5_K;
             }
             if (convert_incompatible_tensor) {
-                new_type = GGML_TYPE_Q8_0; //fall back to Q8_0 instead of just failing.
+                if (tensor.name == "output.weight") {
+                    new_type = GGML_TYPE_F16; //fall back to F16 instead of just failing.
+                    fprintf(stderr, "F16 will be used for this tensor instead.\n");
+                } else if (tensor.name == "tok_embeddings.weight") {
+                    new_type = GGML_TYPE_Q4_0; //fall back to Q4_0 instead of just failing.
+                    fprintf(stderr, "Q4_0 will be used for this tensor instead.\n");
+                } else {
+                    throw std::runtime_error("Unsupported tensor size encountered\n");
+                }
             }
 #endif