vbatts/llama.cpp
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

convert : add convert-hf-to-gguf-update.py

Browse source 
ggml-ci
This commit is contained in:
Georgi Gerganov 2024-04-28 20:29:32 +03:00
parent ee6d1b3fb4
commit 7642973616
No known key found for this signature in database
GPG key ID: 449E073F9DC10735
5 changed files with 215 additions and 26 deletions
Download patch file Download diff file Expand all files Collapse all files

43
convert-hf-to-gguf.py
View file

@ -11,6 +11,7 @@ import sys
from abc import ABC, abstractmethod
from enum import IntEnum
from pathlib import Path
from hashlib import sha256
from typing import TYPE_CHECKING, Any, Callable, ContextManager, Iterator, Sequence, TypeVar, cast
import numpy as np
@ -376,16 +377,19 @@ class Model(ABC):
return tokens, toktypes, tokpre
# NOTE: this function is generated by convert-hf-to-gguf-update.py
# do not modify it manually!
# ref: https://github.com/ggerganov/llama.cpp/pull/6920
def get_vocab_base_pre(self, tokenizer) -> str:
# encoding this string and hashing the resulting tokens would (hopefully) give us a unique identifier that
# is specific for the BPE pre-tokenizer used by the model
# we will use this unique identifier to write a "tokenizer.ggml.pre" entry in the GGUF file which we can
# use in llama.cpp to implement the same pre-tokenizer
chktxt = "\n \n\n \n\n\n \t \t\t \t\n \n \n \n \n🚀 (normal) 😶‍🌫️ (multiple emojis concatenated) ✅ 🦙🦙 3 33 333 3333 33333 333333 3333333 33333333 3.3 3..3 3...3 កាន់តែពិសេសអាច😁 ?我想在apple工作1314151天～ ------======= нещо на Български what's ''''''```````\"\"\"\"......!!!!!!??????"
chktxt = '\n \n\n \n\n\n \t \t\t \t\n \n \n \n \n🚀 (normal) 😶\u200d🌫️ (multiple emojis concatenated) ✅ 🦙🦙 3 33 333 3333 33333 333333 3333333 33333333 3.3 3..3 3...3 កាន់តែពិសេសអាច😁 ?我想在apple工作1314151天～ ------======= нещо на Български what\'s \'\'\'\'\'\'```````""""......!!!!!!??????'
chktok = tokenizer.encode(chktxt)
chkhsh = hash(tuple(chktok))
chkhsh = sha256(str(chktok).encode()).hexdigest()
print(f"chktok: {chktok}")
print(f"chkhsh: {chkhsh}")
@ -393,21 +397,34 @@ class Model(ABC):
res = None
# NOTE: if you get an error here, you need to add the model to the if-elif chain below
# observe the stdout for the chkhsh value and add it to the chain
if self.model_arch == gguf.MODEL_ARCH.LLAMA:
if chkhsh == -3290901550109860290:
# ref: https://huggingface.co/meta-llama/Meta-Llama-3-8B-Instruct/blob/main/tokenizer.json
res = "llama3"
if chkhsh == 5332289095291046364:
# ref: https://huggingface.co/deepseek-ai/deepseek-llm-7b-chat/blob/main/tokenizer.json
res = "deepseek-llm"
if chkhsh == 4190561703949727616:
# ref: https://huggingface.co/deepseek-ai/deepseek-coder-6.7b-instruct/blob/main/tokenizer.json
res = "deepseek-coder"
if chkhsh == "0fc850edd52197e357970116fbf58f6c2567f259cdc1bfc3df081d7e4bc658c1":
# ref: https://huggingface.co/meta-llama/Meta-Llama-3-8B
res = "llama-v3"
if chkhsh == "58c3d0e812ae7fa6a20931006d2398274732c105a9a964c148c43cf898c5fb7a":
# ref: https://huggingface.co/deepseek-ai/deepseek-llm-7b-chat
res = "deepseek-llm"
if chkhsh == "0438d2a948d7fb26c7a662705ac68374f3138ee29e44f133b1f059203500fb4d":
# ref: https://huggingface.co/deepseek-ai/deepseek-coder-6.7b-base
res = "deepseek-coder"
if chkhsh == "406f3f61e1c67d7b0456c5df2fce5cbb30c77dd3671a436b07a6c510303f721e":
# ref: https://huggingface.co/BAAI/bge-small-en-v1.5
res = "bert-bge"
if res is None:
print(f"\n")
print(f"**************************************************************************************")
print(f"** WARNING: The BPE pre-tokenizer was not recognized!")
print(f"** This means that it was not added yet or you are using an older version.")
print(f"** Check convert-hf-to-gguf-update.py and update it accordingly.")
print(f"**")
print(f"** chkhsh: {chkhsh}")
print(f"**************************************************************************************")
print(f"\n")
raise NotImplementedError("BPE pre-tokenizer was not recognized - update get_vocab_base_pre()")
print(f"tokenizer.ggml.pre: {res}")
print(f"chkhsh: {chkhsh}")
return res
def _set_vocab_gpt2(self) -> None: