From 56e149d627e95fbd227580c2b9a6468b35e16623 Mon Sep 17 00:00:00 2001
From: Yutong Dai <rothdyt@gmail.com>
Date: Fri, 4 Oct 2024 22:38:39 +0000
Subject: [PATCH] add quantize method

---
 examples/xgenmm/quantize.sh |   4 ++--
 examples/xgenmm/run_cli.sh  |   7 ++++---
 xgenmm-cli                  | Bin 48440544 -> 48440544 bytes
 3 files changed, 6 insertions(+), 5 deletions(-)

diff --git a/examples/xgenmm/quantize.sh b/examples/xgenmm/quantize.sh
index afc9b9041..d776c7519 100644
--- a/examples/xgenmm/quantize.sh
+++ b/examples/xgenmm/quantize.sh
@@ -5,6 +5,6 @@ quantize_method=Q4_K_M
 outname=${model_name}_${quantize_method}
 input_model_path=$gguf_dir/$model_name.gguf
 output_model_path=$gguf_dir/$outname.gguf
-echo $outname
-cd ../../
+echo $input_model_path
+echo $output_model_path
 ./llama-quantize $input_model_path $output_model_path $quantize_method
\ No newline at end of file
diff --git a/examples/xgenmm/run_cli.sh b/examples/xgenmm/run_cli.sh
index df10fa319..9ae1ac18e 100644
--- a/examples/xgenmm/run_cli.sh
+++ b/examples/xgenmm/run_cli.sh
@@ -35,12 +35,12 @@ make xgenmm-cli
 #     --seed 42 --ctx-size 4096 --predict 1024 \
 #     --temp 0 --verbose-prompt
 
-Q="What is the address of this resturant?"
+Q="What is the address of this restaurant?"
 # Q="Is this dine in or dine out receipt?"
 # Q="What is the total amount paid?"
 # Q="What is card holder's name?"
 # Q="What is the transaction date?"
-# Q="What is the phone number of this resturant?"
+# Q="What is the phone number of this restaurant?"
 # Q="Who is the attendant?"
 # Q="Who is the cashier?"
 # Q="Briefly describe this image."
@@ -53,7 +53,8 @@ echo $prompt
 # mmproj=$base_path/mmproj-model-f32.gguf
 
 base_path=/export/share/yutong/xgenmm/llamacpp_wd/siglip_kosmos_phi3_4k_instruct_bf16_patch128/gguf
-model=$base_path/phi3_mini_4k_instruct_f16.gguf
+# model=$base_path/phi3_mini_4k_instruct_f16.gguf
+model=$base_path/phi3_mini_4k_instruct_f16_Q4_K_M.gguf
 mmproj=$base_path/mmproj-model-f32.gguf
 
 ./xgenmm-cli --model $model\
diff --git a/xgenmm-cli b/xgenmm-cli
index f17b35a8e3091380b562b3c6fe1032f008a93cfc..dac512a93e9672fbc0a6d92fcfa3bbaaad551bcf 100755
GIT binary patch
delta 3493
zcmWmE<9iSa07mg=TefZ6SjKWK+gP^ERm--qti@$6+pg`}M%VY859jy1f597jb3w*f
zii!$KSL5LG51q61xU#p!?C_AfxyD`{cBff*Oi)BjPyhvzJ08WS1eA~xQDRC$Nhuj6
zrxcWuQc-G3Lun};rKb#(kup(c%0gKw8)c^)l#_B%ZpuS>DIevh0#uL+QDG_)9TQY^
z@Rmfq%LMld3#%9$Q7N)06{F%*BDzv!$+Xo1@<xQlm84Qsn#xdFDo5q10#&47szjBk
z3RR_QRGn&2O{zt;DTL}!U8+a*sR1>lM%0*^P$)H}X4IToP)lk>t*H&QrFPVwI#5UI
zM4hP%b)|09oqAAD>P2DHoBB{+>PP))01c!;G?<3aP#Q+VX#|a=Q8b#y&{!Hr<7om-
zq)9ZHrqEQHM$>5qh0{!$MYAb_=FnW4NAqa`Eu=-Xn3m8|T1Lxh1+Ao2w3^n?T3Scz
zX#;JfO|+S|&{o<;+bNQE&`#P#yJ-*YrG2!Y4$wh5L{W5@j?hs$M#t#{Mbk+-MW^Wu
zouzYho-WWux<r@h3dPV>x<=RO2Hm7vberzbUAjm2=>a{YNA#GU&{KLw&*=rdq*wHs
z-q2fmNAD??;^-s#m_DIT=`;GAzMwDZEBczgp>OFs`ksEEAL%FhnSP;P={NeFKF}ZZ
zC;df#(?9et{Wm`}E?_Z9pae-gi7yExp(K*Tl0=e9GD$8eB&DR1)RIQhN;*j|86=}*
zlFX7tvPw3|E;%Hp<dWQyNAgNO$u9+@pcInAQbdYMF)1!3q@<LR(o#msN;xSn6{MmB
zOC_l+Rivs^lj>4KYDz7sEg@1z>PkJSFAb!jG?K>BL_(#hG?V7iLRv~IX)SG}t+bQ&
z(m^^(C+RF*q^oq3?$SegN-qhM-qJ_<N<Zl@17x5KlEE@WhRQG*E+b^5jFQnZM#joG
z87~uLqD+#>GDW7!G?^|lBwS|7ESW74GDqghJee;GWT7mQ#j-?}$}(9lD`cgtlGU<C
z*2+3rFB@c|Y?95gMYhT|*)EZ?Lw3q8*)4lyuk4fkazGBsA&HX1azu{GF*z<LBw9|&
zDLE}?<gA>N^KwBh$|bohS0qNR$~CzzH{_<=lG}1e?#ey6FAwCQJd(%qM4rksc`h&H
zrM!~Y@<!guJ9#g$5+@(Y$MT7MDxb;c@`ZdUU&+_<jeINL$@lVu{3t)k&+?1>D!<9^
z@<INPKjkm^TmF%M<-Y*#FOU%+ff6L~B)%k&gpx=SOA<*c$t1a?kd%^2QcD_1E9oS?
zWRQ%KNis_o$tu|-yX26Zl1p++9?2{DB)=4pf>KBdOA#q5#iY2DkdjhLN=q3jE9IoT
zRFH}iES03PRFSGuO{z-`sVTLjwuDF>sVnuQzBG`A(nuOh6A6{3(oC953u!5>q_wn>
zw$e`8O9$yFouspLk*?BBx=Rn~DZM02dP^VaEB&Ou43L2`NCwLg87jkMxQvjIGD=3v
z7#S<$WV}p}i84th%M_U^(`35LkZ_qPvt+hJ$Q+p~^JKm(kcF~H7RwS@D$8WKtdNzm
zN><AnSu5*gy=;(;vPm|}7TGG>WV=Ml4%sQYWVh^*y|PdC%K<qkha^f4%Mm#$$K<%2
zkZ3t6r{uJpk+X76&dUY4D3|22T#*>LD%a$?+>o1cOK!^@xhwbNzC4hJ@<<-b6L~7n
d<hi_%m-0$p%Nuzs@8rG2N?hOt?k{k8$p5u1Xp8^=

delta 3493
zcmWmE<9iSa07mg=TefYxmY3~i+qUgmwvA;iE_2y#aqHeLySDE+AI|T2|AIH>=7RJw
z6cG`WrsI%_2X{6Z{UQ4JnUB$zFU46D^rG&JsGzW@pa2RacN~gK@hCnepoEl&5>paN
zO35fWrJ$6Qic(V=N=xY|J!PPbl!-D^7RpN5C_Ck#oRo`lQy$7o`6xdXpn_B=GAgL>
zpe+f?_XsW*(z|qUSV(wbDndo6SY$|e@zj+Aa)pJ)7N-&vOeLulm8LROmda6i3ZV*A
zkt$JTszOz%8dawnRFi5^ZK^|csUFp*2Go!mQ7AR0Ce)OgQFCfREvXf?rZ&`;+EIJz
zKpm+Qb*3)VmAX-P>OnoJ7xkt-)R+2Ee;PmoX%G#jAvBbR(Qq0;BWV<krZF^@#?g41
zKoe;aO{OU{m8Q{jnn5#Z7R{zGnnQDG9?ho(w2&6jVp>8=X&Ei26||C8(P~;lYiS*=
zrwz1`HqmC<LR)DYZKrVBK|5&|?WR4nm-f+qIzR{M5Jk{oIzmV37#*h*6iFxP6rH9s
zbe7K1dAdLs=@MP0D-=an=^9<98+4Oy(QUdzcj+G8rw8<q9?@fZLQm-#J*O8GO)u#c
zy{0$xmflee#nOBFgg&JY^cj6lU(lEI6@5(~=^Of%zN7Ey2l|nIqMzv(`jvj8-{}wf
zlm4Q==^y%+{+k~f8?cxpP=X|m#FcmwUlK?{NhFCSi6oU|l3Y?qN=YTDC5@z&bdp{&
zNJhydnI(&4m28q-a!5|eCAlS!<duAqUkXS;DI|rZh!mA#Qd~+%u#}WiQd-JLSt%#w
zB}6JnMX4l}rHWLQYEoTlNKL6FwWW^Km3mTN8c0KFB%#t+nn+V=Ce5XVw3JrTTG~ik
zX(#QagLITm(pkDlSLr6*rHAyCUea6oNMGqE{bhg*ltD6BhR9GECc|ZfjFeF_TE@s&
z87Jdqf=rZ2GFhg`RGB8zWroa@Su$I~WRA?0c`{!X$U<2pi)D!{m1VMAR>(?OC97qP
ztd(`LUN*=^*(94~i)@u`vR%SuhwPMHvRn4ZUfC!6<$xTNLlPl}<%k@WV{%+hNTi&U
zQ*v6)$XPij=jDQ2luL41u1J(zm1}ZcZpcl!CAZ~{+?9KBUmnOqc_feJi9D5O@?2g>
zw7is8@><@=TX`oj5-acJ6Zup=$Y=7od?8=TSMs%dlyBr)`A)u<ALK{*Nq&}J<X8Dk
zewRPwPx(v!mVe}5`7eO`3uFXHpae-Ai7W9Wz9f)@l1LIu5=kn_B)O!Jl#)tPOBzWl
z=_I{mkc^T^GD{Z8D%m8v<dB?_OL9vd$t(FJzZ8&yQb-C*5h*Igq_~ujU@0l3q_mWg
zvQkdUONdmEic(1`OBJaq)ug)AkeX6UYD*odEA^zlG?0eUNJ6EtG?Av#OqxpzX(_Fw
zwX~78(oWh-2k9uCq_cF9uF_4qOAqNOy`;DFk-pMT`pW<rD1&6M43VKSOoqz{87ZS=
zw2YClGET<J1eqw4WU@?=sWMHb%M6(*vt+h}$sCz0^JKm(kcF~H7RwS@D$8WKtdNzm
zN><AnSu5*gy=;(;vPm|}7TGG>WV?jR4%sQYWVh^*y|PdC%K<qkha^G{%Mm#$$K<%2
zkVrWxr{uJpk+X76&dUY4D3|22T#+cbD%a$?+>o1cOK!^@xhwbNzC4hJ@<<-b6L~7n
d<hi_%Xn85G<h8t!xAIP6BsTB@_ZPUl=KpQyY3Kj|