From 8c179f41faf7b2c30aa9635f21cfc9e09b14c625 Mon Sep 17 00:00:00 2001 From: Yutong Dai Date: Wed, 2 Oct 2024 23:32:35 +0000 Subject: [PATCH] get a new ckpt --- examples/xgenmm/convert.sh | 61 ++++++++++++++++++++++++++++--------- examples/xgenmm/run_cli.sh | 19 ++++++++---- xgenmm-cli | Bin 48440544 -> 48440544 bytes 3 files changed, 59 insertions(+), 21 deletions(-) diff --git a/examples/xgenmm/convert.sh b/examples/xgenmm/convert.sh index a2829a5d3..c8fe8ba50 100644 --- a/examples/xgenmm/convert.sh +++ b/examples/xgenmm/convert.sh @@ -1,16 +1,48 @@ source /export/share/yutong/miniconda3/bin/activate conda activate xgenmm-flamingo -# which python -# # step 1: surgery -# python xgenmm_surgery.py +which python -# # step 2: convert vit + projector to gguf +# ======= siglip_kosmos_phi3_4k_instruct ======= -# python xgenmm_convert_image_encoder_to_gguf.py \ -# --surgery_dir /export/share/yutong/xgenmm/llamacpp_wd \ -# --output_dirname gguf_test \ -# --version siglip_kosmos_phi3_4k_instruct \ -# --use_f32 +# # # step 1: surgery +# # python xgenmm_surgery.py + +# # # step 2: convert vit + projector to gguf + +# # python xgenmm_convert_image_encoder_to_gguf.py \ +# # --surgery_dir /export/share/yutong/xgenmm/llamacpp_wd \ +# # --output_dirname gguf \ +# # --version siglip_kosmos_phi3_4k_instruct \ +# # --use_f32 + +# # step 3: convert llm to gguf +# # https://github.com/ggerganov/llama.cpp/discussions/7927 +# cd ../../ +# # HF_TOKEN= +# # downloads the tokenizer models of the specified models from Huggingface; generates the get_vocab_base_pre() function for convert_hf_to_gguf.py +# # python convert_hf_to_gguf_update.py $HF_TOKEN + + +# LLM_PATH=/export/share/yutong/xgenmm/llamacpp_wd/siglip_kosmos_phi3_4k_instruct/llm +# outtype=f32 +# LLM_OUTPUT_FILE=/export/share/yutong/xgenmm/llamacpp_wd/siglip_kosmos_phi3_4k_instruct/gguf/phi3_mini_4k_instruct_$outtype.gguf +# echo $LLM_OUTPUT_FILE +# python convert_hf_to_gguf.py $LLM_PATH --outfile $LLM_OUTPUT_FILE --outtype $outtype + + +# ======= siglip_kosmos_phi3_4k_instruct_bf16_patch128 ======= + +CKPT_PATH=/export/share/manli_shu/models/open-flamingo-dev/fixed_offset-bf16-maxlen2048-newsamplerv1-anyres_patch128-kosmos_non_instruct-phi3_4k_instruct_nq128_pre_V3_6-SFT_v3.6.1.v2-mantis-mix-v0.3.5-continue-8x16-ckpt0/checkpoint_0.pt +VERSION=siglip_kosmos_phi3_4k_instruct_bf16_patch128 +SAVE_PATH=/export/share/yutong/xgenmm/llamacpp_wd +# step 1: surgery +python xgenmm_surgery.py --ckpt_pth $CKPT_PATH --save_pth $SAVE_PATH --version $VERSION +# step 2: convert vit + projector to gguf +python xgenmm_convert_image_encoder_to_gguf.py \ + --surgery_dir $SAVE_PATH\ + --output_dirname gguf \ + --version $VERSION \ + --use_f32 # step 3: convert llm to gguf # https://github.com/ggerganov/llama.cpp/discussions/7927 @@ -19,9 +51,8 @@ cd ../../ # downloads the tokenizer models of the specified models from Huggingface; generates the get_vocab_base_pre() function for convert_hf_to_gguf.py # python convert_hf_to_gguf_update.py $HF_TOKEN - -LLM_PATH=/export/share/yutong/xgenmm/llamacpp_wd/siglip_kosmos_phi3_4k_instruct/llm -outtype=f32 -LLM_OUTPUT_FILE=/export/share/yutong/xgenmm/llamacpp_wd/siglip_kosmos_phi3_4k_instruct/gguf/phi3_mini_4k_instruct_$outtype.gguf -echo $LLM_OUTPUT_FILE -python convert_hf_to_gguf.py $LLM_PATH --outfile $LLM_OUTPUT_FILE --outtype $outtype \ No newline at end of file +# go to llm folder and nano config.json change vocab_size to 32064 +LLM_PATH=$SAVE_PATH/$VERSION/llm +OUTTYPE=f16 +LLM_OUTPUT_FILE=$SAVE_PATH/$VERSION/gguf/phi3_mini_4k_instruct_$OUTTYPE.gguf +python convert_hf_to_gguf.py $LLM_PATH --outfile $LLM_OUTPUT_FILE --outtype $OUTTYPE \ No newline at end of file diff --git a/examples/xgenmm/run_cli.sh b/examples/xgenmm/run_cli.sh index dcbee799c..5da2bd376 100644 --- a/examples/xgenmm/run_cli.sh +++ b/examples/xgenmm/run_cli.sh @@ -41,17 +41,24 @@ make xgenmm-cli # Q="What is card holder's name?" # Q="What is the transaction date?" # Q="What is the phone number of this resturant?" -Q="Who is the attendant?" -# Q="Who is the cashier?" +# Q="Who is the attendant?" +Q="Who is the cashier?" # Q="Briefly describe this image." prompt="<|system|>\nA chat between a curious user and an artificial intelligence assistant. The assistant gives helpful, detailed, and polite answers to the user's questions.<|end|>\n<|user|>\n\n $Q<|end|>\n<|assistant|>\n" echo $prompt -model=/export/share/yutong/xgenmm/llamacpp_wd/siglip_kosmos_phi3_4k_instruct/gguf/phi3_mini_4k_instruct_f32.gguf -# model=/export/share/yutong/xgenmm/llamacpp_wd/siglip_kosmos_phi3_4k_instruct/gguf/phi3_mini_4k_instruct_f16.gguf +# base_path=/export/share/yutong/xgenmm/llamacpp_wd/siglip_kosmos_phi3_4k_instruct/gguf +# # model=$base_path/phi3_mini_4k_instruct_f32.gguf +# model=$base_path/phi3_mini_4k_instruct_f16.gguf +# mmproj=$base_path/mmproj-model-f32.gguf + +base_path=/export/share/yutong/xgenmm/llamacpp_wd/siglip_kosmos_phi3_4k_instruct_bf16_patch128/gguf +model=$base_path/phi3_mini_4k_instruct_f16.gguf +mmproj=$base_path/mmproj-model-f32.gguf + ./xgenmm-cli --model $model\ - --mmproj /export/share/yutong/xgenmm/llamacpp_wd/siglip_kosmos_phi3_4k_instruct/gguf_test/mmproj-model-f32.gguf \ + --mmproj $mmproj \ --image /export/home/llama.cpp/examples/xgenmm/imgs/receipt.jpg\ --prompt "$prompt" \ --seed 42 --ctx-size 4096 --predict 1024 \ - --temp 0.8 --verbose-prompt --color --ubatch-size 1280 \ No newline at end of file + --temp 0.0 --verbose-prompt --color --ubatch-size 1280 \ No newline at end of file diff --git a/xgenmm-cli b/xgenmm-cli index 2d1a1e4303613a8a2a4dd5261e01c1a6b9cc37e5..ef1407df10c89704ca0525f7b0be144d32c1d292 100755 GIT binary patch delta 3493 zcmWmE<9iSa07mg=TefZ6wU+H=+qUgmwQO@)3u~Fnwz00!Mc2OPd^o@7{R`fA*XO5y zN0E_1X?A^_6;&hlQPif%K_BuoE*3trcGhMyVuHeAf&wUz+;J!_#iRI?fD%$7N=!*8 zDJ7%il!8)HDoRahC@rO<^pt@zQYOkwStu)IqwJJ}a#AkJO?fCU<)i#mfC^Hf=$N3w zgEl896&zB!MEM@UVIdKPsR$LNV$mTH#Zy-f$Qc$ITbxQzNh(F9sSK5+auiJEDTFFe zMXE%VsR~u4YE+$SP)({uwW$u(rFvAK8c;)OM4{A}nov_}M$M@OwWLUeufVP+#gt{b>LVq(L;8hR{$NM#E_YjigaDn#Ryr8b{-4 z0!^ezG?}K*RGLQ9X$H-tSu~r%Xb#P#c@$3bX#p*yMYNcf&{A4P%V`Czq*b(<*3ep7 zN9$<=ZKO@KnYPeY+C~wyop#Vp+C{r*5ACIWw4V;pL5ie9beN9NQHr8t6ivtJ1f8T) zbehi4Svp7O=>lD(OB6$w=?Yz?YjmA%&`r8Ux9JYurF(Rr9?(O2M33nSJ*8*#oL@BwCKk z2{|dJZKnapK5?A6$d`Tb)C6OeSB$8B;NpeXcDJ7MpmNb%9 z(n)&BAQ>f-WR@(FRkBHT$ssuhNESD9s zQdY@oStDy@ovfD)vQak4X4xWJWt&9EcG)32WtZ%hJ+fE!$$mK?2PIMt$zeGnMHJ5b9o^z<(0gaH}Y2ANo?Rb?k{jz&Hn*v8fXjv delta 3493 zcmWmEV|y3|07c;^Np0J%Ql@5V+ilWqo!fbxcg~0NKKB>g=<8u= zqbVXHC{^E_JNAYxs-N)F$X8R|XSo>jbaIZ|)1!hyqk;k`kle8-HpQX16p!Ll0!m1U zC^03Wq?C-3QwmB+sVFt2p|q5a(o+V?NSP=zWudH;jj~e?%1OB>H|3$cl#lXLfyk(! zf&(|jD_N{t+0w;(_6iLSFGz)`Fcpam4lkOrQb4xQkeH%WjEYkUDoLfNG?k&URE~nF zJXN5IREa866{H^{75IpoSDeji@m-p{CS~no|pENv)_gwV}4u zj@nZP>PVfaGj*Y^)Q!4R59&$1s5kYYzSNKU(*PPsgJ>`fp`kR4hSLZdNuy{qjiIqL zj>gjjnn;sqGEJeWG>xXy44O%^D3oT?9GXk>D2(RQ0$NCmXfZ9JrL>He(+XNit7tW? zp|!M**3$;sNSkOgZK16cPTOcZ?Vz2si+0l<+DrRrKOLY5I!K4;Fddwx9B$Ap}TaC?$ZN$NRQ|-J)x)cjGogA zdP%S7HNBy?6iqSojy|K$=?i*KU(#3fHGM1Qc_AuX(=OR zrJMvyd8r^3rIJ*ZDpFOdNp-0qHKmr+mO4^b>PdZRAPpr%8cAbmB2A^4G?y0AQd&uC zX(Mf=owSz@(os4|XXzqcrJHn@9@0~KNpI;ReWjoDmjN^8vBWq=ytd|Y4Q8vkD*&P3GjdkW$$7aT7v+*f$z{1BSLK>qmm6|ZZpm%ABX{MV+?NOPP#(!+c_L5c znLL*l@={*OYk4DYC0b(SoqQ&r%NO!qzLc-zYxzdLmG9(x`5+(V2l-KclAq-l`Bi?C z-{q71A%DtW^0)jW|H^*>++QFgKmsL5Vo7X?BXK33#Fqq;P!dUENg_!lnIxAKl2TGh zYDpt$C7q;~43beYNoL6+StXlfmmHE)a!GE1Qc_Au zX(=ORrJMvyd8r^3rIJ*ZDpFOdNp-0qHKmr+mO4^b>PdZRAPpr%8cAbmB2A^4G?y0A zQd&uCX(Mf=owSz@(os4|XXzqcrJHn@9@0~KNpI;ReWjoDmjN^8vBWq=ytd|Y4Q8vkD*&P3GjdkW$$7aT7v+*f$z{1BSLK>qmm6|ZZpm%ABX{MV+?NOPP#(!+ fc_L5cnLL*l@={*OYk4DYC0b$v&vAc&OKbcOL}zH<