From 1ec79f04ab4854be0ccbad768f2e9325e01b5188 Mon Sep 17 00:00:00 2001 From: caitianchi Date: Mon, 12 Aug 2024 20:17:56 +0800 Subject: [PATCH] modify convert script and readme --- examples/llava/README-minicpmv2.5.md | 4 +- examples/llava/README-minicpmv2.6.md | 2 +- ...cpmv2_6-surgery.py => minicpmv-surgery.py} | 3 +- .../minicpmv-convert/minicpmv2_5-surgery.py | 46 ------------------- 4 files changed, 4 insertions(+), 51 deletions(-) rename examples/llava/minicpmv-convert/{minicpmv2_6-surgery.py => minicpmv-surgery.py} (94%) delete mode 100644 examples/llava/minicpmv-convert/minicpmv2_5-surgery.py diff --git a/examples/llava/README-minicpmv2.5.md b/examples/llava/README-minicpmv2.5.md index 9e011965d..166b9e2e9 100644 --- a/examples/llava/README-minicpmv2.5.md +++ b/examples/llava/README-minicpmv2.5.md @@ -15,8 +15,8 @@ cd llama.cpp Convert PyTorch model to gguf files (You can also download the converted [gguf](https://huggingface.co/openbmb/MiniCPM-Llama3-V-2_5-gguf) by us) ```bash -python ./examples/minicpmv/minicpmv-surgery.py -m ../MiniCPM-Llama3-V-2_5 -python ./examples/minicpmv/minicpmv-convert-image-encoder-to-gguf.py -m ../MiniCPM-Llama3-V-2_5 --minicpmv-projector ../MiniCPM-Llama3-V-2_5/minicpmv.projector --output-dir ../MiniCPM-Llama3-V-2_5/ --image-mean 0.5 0.5 0.5 --image-std 0.5 0.5 0.5 +python ./examples/minicpmv/minicpmv-convert/minicpmv-surgery.py -m ../MiniCPM-Llama3-V-2_5 +python ./examples/minicpmv/minicpmv-convert/minicpmv2_5-convert-image-encoder-to-gguf.py -m ../MiniCPM-Llama3-V-2_5 --minicpmv-projector ../MiniCPM-Llama3-V-2_5/minicpmv.projector --output-dir ../MiniCPM-Llama3-V-2_5/ --image-mean 0.5 0.5 0.5 --image-std 0.5 0.5 0.5 python ./convert_hf_to_gguf.py ../MiniCPM-Llama3-V-2_5/model # quantize int4 version diff --git a/examples/llava/README-minicpmv2.6.md b/examples/llava/README-minicpmv2.6.md index ecd13d362..6af84abb6 100644 --- a/examples/llava/README-minicpmv2.6.md +++ b/examples/llava/README-minicpmv2.6.md @@ -16,7 +16,7 @@ git checkout minicpmv-main Convert PyTorch model to gguf files (You can also download the converted [gguf](https://huggingface.co/openbmb/MiniCPM-V-2_6-gguf) by us) ```bash -python ./examples/llava/minicpmv-convert/minicpmv2_6-surgery.py -m ../MiniCPM-V-2_6 +python ./examples/llava/minicpmv-convert/minicpmv-surgery.py -m ../MiniCPM-V-2_6 python ./examples/llava/minicpmv-convert/minicpmv2_6-convert-image-encoder-to-gguf.py -m ../MiniCPM-V-2_6 --minicpmv-projector ../MiniCPM-V-2_6/minicpmv.projector --output-dir ../MiniCPM-V-2_6/ --image-mean 0.5 0.5 0.5 --image-std 0.5 0.5 0.5 python ./convert_hf_to_gguf.py ../MiniCPM-V-2_6/model diff --git a/examples/llava/minicpmv-convert/minicpmv2_6-surgery.py b/examples/llava/minicpmv-convert/minicpmv-surgery.py similarity index 94% rename from examples/llava/minicpmv-convert/minicpmv2_6-surgery.py rename to examples/llava/minicpmv-convert/minicpmv-surgery.py index 34e7d794e..748ff5c57 100644 --- a/examples/llava/minicpmv-convert/minicpmv2_6-surgery.py +++ b/examples/llava/minicpmv-convert/minicpmv-surgery.py @@ -4,7 +4,7 @@ import torch from transformers import AutoModel, AutoTokenizer ap = argparse.ArgumentParser() -ap.add_argument("-m", "--model", help="Path to MiniCPM-V-2.6 model") +ap.add_argument("-m", "--model", help="Path to MiniCPM-V model") args = ap.parse_args() # find the model part that includes the the multimodal projector weights @@ -29,7 +29,6 @@ if len(clip_tensors) > 0: f.write("{}\n") config = model.llm.config -config._name_or_path = "openbmb/MiniCPM-V-2.6" config.auto_map = { "AutoConfig": "configuration_minicpm.MiniCPMConfig", "AutoModel": "modeling_minicpm.MiniCPMModel", diff --git a/examples/llava/minicpmv-convert/minicpmv2_5-surgery.py b/examples/llava/minicpmv-convert/minicpmv2_5-surgery.py deleted file mode 100644 index 7d6baa8af..000000000 --- a/examples/llava/minicpmv-convert/minicpmv2_5-surgery.py +++ /dev/null @@ -1,46 +0,0 @@ -import argparse -import os -import torch -from transformers import AutoModel, AutoTokenizer - -ap = argparse.ArgumentParser() -ap.add_argument("-m", "--model", help="Path to MiniCPM-V-2.5 model") -args = ap.parse_args() - -# find the model part that includes the the multimodal projector weights -model = AutoModel.from_pretrained(args.model, trust_remote_code=True, local_files_only=True) -checkpoint = model.state_dict() - -# get a list of mm tensor names -mm_tensors = [k for k, v in checkpoint.items() if k.startswith("resampler")] - -# store these tensors in a new dictionary and torch.save them -projector = {name: checkpoint[name].float() for name in mm_tensors} -torch.save(projector, f"{args.model}/minicpmv.projector") - -clip_tensors = [k for k, v in checkpoint.items() if k.startswith("vpm")] -if len(clip_tensors) > 0: - clip = {name.replace("vpm.", ""): checkpoint[name].float() for name in clip_tensors} - torch.save(clip, f"{args.model}/minicpmv.clip") - - # added tokens should be removed to be able to convert Mistral models - if os.path.exists(f"{args.model}/added_tokens.json"): - with open(f"{args.model}/added_tokens.json", "w") as f: - f.write("{}\n") - -config = model.llm.config -config._name_or_path = "openbmb/MiniCPM-Llama3-V-2.5" -config.auto_map = { - "AutoConfig": "configuration_minicpm.MiniCPMConfig", - "AutoModel": "modeling_minicpm.MiniCPMModel", - "AutoModelForCausalLM": "modeling_minicpm.MiniCPMForCausalLM", - "AutoModelForSeq2SeqLM": "modeling_minicpm.MiniCPMForCausalLM", - "AutoModelForSequenceClassification": "modeling_minicpm.MiniCPMForSequenceClassification" -} -model.llm.save_pretrained(f"{args.model}/model") -tok = AutoTokenizer.from_pretrained(args.model, trust_remote_code=True) -tok.save_pretrained(f"{args.model}/model") - -print("Done!") -print(f"Now you can convert {args.model} to a regular LLaMA GGUF file.") -print(f"Also, use {args.model}/minicpmv.projector to prepare a minicpmv-encoder.gguf file.")