py : add Phi-1.5/Phi-2 tokenizer (#9361)
* add phi2 tokenizer * add phi name to convert_hf_to_gguf_update.py * make tokenizer_pre consistent; llama.cpp work
This commit is contained in:
parent
3c26a1644d
commit
c837981bba
2 changed files with 4 additions and 0 deletions
|
@ -626,6 +626,9 @@ class Model:
|
|||
if chkhsh == "4e2b24cc4770243d65a2c9ec19770a72f08cffc161adbb73fcbb6b7dd45a0aae":
|
||||
# ref: https://huggingface.co/LGAI-EXAONE/EXAONE-3.0-7.8B-Instruct
|
||||
res = "exaone"
|
||||
if chkhsh == "fcace8b9cac38ce847670c970cd5892031a753a1ef381abd1d9af00f713da085":
|
||||
# ref: https://huggingface.co/microsoft/phi-2
|
||||
res = "phi-2"
|
||||
|
||||
if res is None:
|
||||
logger.warning("\n")
|
||||
|
|
Loading…
Add table
Add a link
Reference in a new issue