vbatts/llama.cpp
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Add vision feature layer to gguf params

Browse source 
Signed-off-by: Alex-Brooks <Alex.Brooks@ibm.com>
This commit is contained in:
Alex-Brooks 2025-01-22 01:27:33 -07:00
parent bc66d1931b
commit 92046a103d
1 changed files with 10 additions and 1 deletions
Download patch file Download diff file Expand all files Collapse all files

11
examples/llava/convert_image_encoder_to_gguf.py
View file

@ -6,7 +6,7 @@ import re
import torch import torch
import numpy as np import numpy as np
from gguf import * from gguf import *
from transformers import CLIPModel, CLIPProcessor, CLIPVisionModel, SiglipModel, SiglipProcessor, SiglipVisionModel from transformers import CLIPModel, CLIPProcessor, CLIPVisionModel, SiglipVisionModel
TEXT = "clip.text" TEXT = "clip.text"
VISION = "clip.vision" VISION = "clip.vision"
@ -208,6 +208,15 @@ if has_text_encoder:
fout.add_token_list(tokens) fout.add_token_list(tokens)
if has_vision_encoder: if has_vision_encoder:
# vision feature layer may be an integer or an array.
# TODO - it seems like llama cpp may not handle this correctly
# normally; check if HF llava next models can run through this converter...
if "vision_feature_layer" in v_hparams:
feature_layers = v_hparams["vision_feature_layer"]
if isinstance(feature_layers, int):
feature_layers = [feature_layers]
fout.add_array("clip.vision.feature_layer", feature_layers)
if args.clip_model_is_siglip: if args.clip_model_is_siglip:
visual_projection_dim = 0 visual_projection_dim = 0
else: else: