Export new tensors in 1D so they are not quantized.

convert-hf-to-gguf.py

@@ -2352,6 +2352,53 @@ class CommandR2Model(Model):
         self.gguf_writer.add_logit_scale(self.hparams["logit_scale"])
         self.gguf_writer.add_rope_scaling_type(gguf.RopeScalingType.NONE)
 
+    def write_tensors(self):
+        block_count = self.hparams.get("n_layers", self.hparams.get("num_hidden_layers", self.hparams.get("n_layer")))
+        tensor_map = gguf.get_tensor_name_map(self.model_arch, block_count)
+        for name, data_torch in self.get_tensors():
+            # we don't need these
+            if name.endswith((".attention.masked_bias", ".attention.bias", ".attention.rotary_emb.inv_freq")):
+                continue
+
+            #Convert Q norm and K norm to 1d so they are exported in float32 and not quantized
+            if name.endswith((".q_norm.weight")) or name.endswith((".k_norm.weight")):
+                data_torch = data_torch.flatten()
+
+            old_dtype = data_torch.dtype
+
+            # convert any unsupported data types to float32
+            if data_torch.dtype not in (torch.float16, torch.float32):
+                data_torch = data_torch.to(torch.float32)
+
+            data = data_torch.squeeze().numpy()
+
+            # map tensor names
+            new_name = tensor_map.get_name(name, try_suffixes=(".weight", ".bias"))
+            if new_name is None:
+                print(f"Can not map tensor {name!r}")
+                sys.exit()
+
+            n_dims = len(data.shape)
+            data_dtype = data.dtype
+
+            # if f32 desired, convert any float16 to float32
+            if self.ftype == 0 and data_dtype == np.float16:
+                data = data.astype(np.float32)
+
+            # TODO: Why cant we use these float16 as-is? There should be not reason to store float16 as float32
+            if self.ftype == 1 and data_dtype == np.float16 and n_dims == 1:
+                data = data.astype(np.float32)
+
+            # if f16 desired, convert any float32 2-dim weight tensors to float16
+            if self.ftype == 1 and data_dtype == np.float32 and name.endswith(".weight") and n_dims == 2:
+                data = data.astype(np.float16)
+
+            print(f"{new_name}, n_dims = {n_dims}, {old_dtype} --> {data.dtype}")
+
+            self.gguf_writer.add_tensor(new_name, data)
+
+
+
 
 ###### CONVERSION LOGIC ######
 

llama.cpp

@@ -5408,8 +5408,8 @@ static bool llm_load_tensors(
                         
                         if(n_layer >= 64)
                         {
-                            layer.attn_q_norm   = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_ATTN_Q_NORM, "weight", i), {hparams.n_embd_head_k, hparams.n_head});
+                            layer.attn_q_norm   = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_ATTN_Q_NORM, "weight", i), {hparams.n_embd_head_k * hparams.n_head});
-                            layer.attn_k_norm   = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_ATTN_K_NORM, "weight", i), {hparams.n_embd_head_k, hparams.n_head_kv});
+                            layer.attn_k_norm   = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_ATTN_K_NORM, "weight", i), {hparams.n_embd_head_k * hparams.n_head_kv});
                         }
                         
                         layer.wq = ml.create_tensor(ctx_split, tn(LLM_TENSOR_ATTN_Q,   "weight", i), {n_embd, n_embd});
@@ -9462,17 +9462,17 @@ struct llm_build_context {
 
                 if(model.layers[il].attn_q_norm)
                 {
-                    Qcur = llm_build_norm(ctx0, Qcur, hparams,
+                    Qcur = llm_build_norm(ctx0, Qcur, hparams, 
                                 model.layers[il].attn_q_norm,
                                 NULL,
                                 LLM_NORM, cb, il);
                     cb(Qcur, "Qcur", il);
-
+                    
                     Kcur = llm_build_norm(ctx0, Kcur, hparams,
                             model.layers[il].attn_k_norm,
                             NULL,
                             LLM_NORM, cb, il);
-                    cb(Kcur, "Kcur", il);
+                    cb(Kcur, "Kcur", il);                  
                 }
 
                 Qcur = ggml_rope_custom(