From cd0f33f281b6f7e20850bac97717205cd4e51d30 Mon Sep 17 00:00:00 2001 From: Francis Couture-Harpin Date: Sun, 4 Feb 2024 09:49:23 -0500 Subject: [PATCH] mamba : fix vocab size problems with official models The perplexity was waaaay to high for models with a non-round vocab size. Not sure why, but it needed to be fixed in the metadata. Note that this breaks existing GGUF-converted Mamba models, but **only if** the vocab size was not already rounded. --- convert-hf-to-gguf.py | 7 +++++++ llama.cpp | 6 ++---- 2 files changed, 9 insertions(+), 4 deletions(-) diff --git a/convert-hf-to-gguf.py b/convert-hf-to-gguf.py index 42b0fb66e..93d714a40 100755 --- a/convert-hf-to-gguf.py +++ b/convert-hf-to-gguf.py @@ -1848,6 +1848,13 @@ class StarCoder2Model(Model): class MambaModel(Model): model_arch = gguf.MODEL_ARCH.MAMBA + def set_vocab(self): + vocab_size = self.hparams["vocab_size"]; + # Round vocab size to next multiple of 8 + pad_vocab = self.hparams.get("pad_vocab_size_multiple", 8); + self.hparams["vocab_size"] = ((vocab_size + (pad_vocab - 1)) // pad_vocab) * pad_vocab + return self._set_vocab_gpt2() + def set_gguf_parameters(self): d_model = self.hparams["d_model"] d_inner = self.hparams.get("d_inner", 2 * d_model) diff --git a/llama.cpp b/llama.cpp index 37ac7425d..0a6ff7736 100644 --- a/llama.cpp +++ b/llama.cpp @@ -4682,15 +4682,13 @@ static bool llm_load_tensors( // FIXME: ceiling instead of floor const int64_t dt_rank = n_embd / 16; GGML_ASSERT(2 * n_embd == d_inner); - // round up the vocab size to the next multiple of 8 - const int64_t rounded_vocab = (n_vocab + 7) & -8; - model.tok_embd = ml.create_tensor(ctx_input, tn(LLM_TENSOR_TOKEN_EMBD, "weight"), {n_embd, rounded_vocab}); + model.tok_embd = ml.create_tensor(ctx_input, tn(LLM_TENSOR_TOKEN_EMBD, "weight"), {n_embd, n_vocab}); // output { model.output_norm = ml.create_tensor(ctx_output, tn(LLM_TENSOR_OUTPUT_NORM, "weight"), {n_embd}); - model.output = ml.create_tensor(ctx_output_split, tn(LLM_TENSOR_OUTPUT, "weight"), {n_embd, rounded_vocab}); + model.output = ml.create_tensor(ctx_output_split, tn(LLM_TENSOR_OUTPUT, "weight"), {n_embd, n_vocab}); } for (int i = 0; i < n_layer; ++i) {