From 50504063b2c74d29e1e5d20ccadf3e9550c39537 Mon Sep 17 00:00:00 2001 From: Alex-Brooks Date: Wed, 22 Jan 2025 01:44:48 -0700 Subject: [PATCH] Add transformers llava next tensor name mapping Signed-off-by: Alex-Brooks --- examples/llava/convert_image_encoder_to_gguf.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/examples/llava/convert_image_encoder_to_gguf.py b/examples/llava/convert_image_encoder_to_gguf.py index cb0495156..d51428c3f 100644 --- a/examples/llava/convert_image_encoder_to_gguf.py +++ b/examples/llava/convert_image_encoder_to_gguf.py @@ -37,6 +37,18 @@ def should_skip_tensor(name: str, has_text: bool, has_vision: bool, has_llava: b def get_tensor_name(name: str) -> str: + # Standardize the transformers llava next keys for + # image newline / mm projector with the classes in haotian-liu LLaVA + if name == "image_newline": + return "model.image_newline" + if name.startswith("multi_modal_projector"): + name = name.replace("multi_modal_projector", "mm") + if name.endswith("linear_1"): + name = name.replace("linear_1", "0") + if name.endswith("linear_2"): + name = name.replace("linear_2", "1") + return name + if "projection" in name: return name if "mm_projector" in name: