convert.py : add python logging instead of print() (#6511)
* convert.py: add python logging instead of print() * convert.py: verbose flag takes priority over dump flag log suppression * convert.py: named instance logging * convert.py: use explicit logger id string * convert.py: convert extra print() to named logger * convert.py: sys.stderr.write --> logger.error * *.py: Convert all python scripts to use logging module * requirements.txt: remove extra line * flake8: update flake8 ignore and exclude to match ci settings * gh-actions: add flake8-no-print to flake8 lint step * pre-commit: add flake8-no-print to flake8 and also update pre-commit version * convert-hf-to-gguf.py: print() to logger conversion * *.py: logging basiconfig refactor to use conditional expression * *.py: removed commented out logging * fixup! *.py: logging basiconfig refactor to use conditional expression * constant.py: logger.error then exit should be a raise exception instead * *.py: Convert logger error and sys.exit() into a raise exception (for atypical error) * gguf-convert-endian.py: refactor convert_byteorder() to use tqdm progressbar * verify-checksum-model.py: This is the result of the program, it should be printed to stdout. * compare-llama-bench.py: add blank line for readability during missing repo response * reader.py: read_gguf_file() use print() over logging * convert.py: warning goes to stderr and won't hurt the dump output * gguf-dump.py: dump_metadata() should print to stdout * convert-hf-to-gguf.py: print --> logger.debug or ValueError() * verify-checksum-models.py: use print() for printing table * *.py: refactor logging.basicConfig() * gguf-py/gguf/*.py: use __name__ as logger name Since they will be imported and not run directly. * python-lint.yml: use .flake8 file instead * constants.py: logger no longer required * convert-hf-to-gguf.py: add additional logging * convert-hf-to-gguf.py: print() --> logger * *.py: fix flake8 warnings * revert changes to convert-hf-to-gguf.py for get_name() * convert-hf-to-gguf-update.py: use triple quoted f-string instead * *.py: accidentally corrected the wrong line * *.py: add compilade warning suggestions and style fixes
This commit is contained in:
Brian
GitHub
parent
433def286e
commit
a2ac89d6ef
23 changed files with 536 additions and 482 deletions
|
|
@ -21,6 +21,7 @@
|
|||
# TODO: automate the update of convert-hf-to-gguf.py
|
||||
#
|
||||
|
||||
import logging
|
||||
import os
|
||||
import requests
|
||||
import sys
|
||||
|
|
@ -28,12 +29,17 @@ import json
|
|||
|
||||
from hashlib import sha256
|
||||
from enum import IntEnum, auto
|
||||
from transformers import AutoTokenizer
|
||||
|
||||
logger = logging.getLogger("convert-hf-to-gguf-update")
|
||||
|
||||
|
||||
class TOKENIZER_TYPE(IntEnum):
|
||||
SPM = auto()
|
||||
BPE = auto()
|
||||
WPM = auto()
|
||||
|
||||
|
||||
# TODO: this string has to exercise as much pre-tokenizer functionality as possible
|
||||
# will be updated with time - contributions welcome
|
||||
chktxt = '\n \n\n \n\n\n \t \t\t \t\n \n \n \n \n🚀 (normal) 😶🌫️ (multiple emojis concatenated) ✅ 🦙🦙 3 33 333 3333 33333 333333 3333333 33333333 3.3 3..3 3...3 កាន់តែពិសេសអាច😁 ?我想在apple工作1314151天～ ------======= нещо на Български \'\'\'\'\'\'```````\"\"\"\"......!!!!!!?????? I\'ve been \'told he\'s there, \'RE you sure? \'M not sure I\'ll make it, \'D you like some tea? We\'Ve a\'lL'
|
||||
|
|
@ -41,36 +47,38 @@ chktxt = '\n \n\n \n\n\n \t \t\t \t\n \n \n \n \n🚀 (normal) 😶
|
|||
if len(sys.argv) == 2:
|
||||
token = sys.argv[1]
|
||||
else:
|
||||
print("Usage: python convert-hf-to-gguf-update.py <huggingface_token>")
|
||||
logger.info("Usage: python convert-hf-to-gguf-update.py <huggingface_token>")
|
||||
sys.exit(1)
|
||||
|
||||
# TODO: add models here, base models preferred
|
||||
models = [
|
||||
{ "name": "llama-spm", "tokt": TOKENIZER_TYPE.SPM, "repo": "https://huggingface.co/meta-llama/Llama-2-7b-hf", },
|
||||
{ "name": "llama-bpe", "tokt": TOKENIZER_TYPE.BPE, "repo": "https://huggingface.co/meta-llama/Meta-Llama-3-8B", },
|
||||
{ "name": "phi-3", "tokt": TOKENIZER_TYPE.SPM, "repo": "https://huggingface.co/microsoft/Phi-3-mini-4k-instruct", },
|
||||
{ "name": "deepseek-llm", "tokt": TOKENIZER_TYPE.BPE, "repo": "https://huggingface.co/deepseek-ai/deepseek-llm-7b-base", },
|
||||
{ "name": "deepseek-coder", "tokt": TOKENIZER_TYPE.BPE, "repo": "https://huggingface.co/deepseek-ai/deepseek-coder-6.7b-base", },
|
||||
{ "name": "falcon", "tokt": TOKENIZER_TYPE.BPE, "repo": "https://huggingface.co/tiiuae/falcon-7b", },
|
||||
{ "name": "bert-bge", "tokt": TOKENIZER_TYPE.WPM, "repo": "https://huggingface.co/BAAI/bge-small-en-v1.5", },
|
||||
{ "name": "mpt", "tokt": TOKENIZER_TYPE.BPE, "repo": "https://huggingface.co/mosaicml/mpt-7b", },
|
||||
{ "name": "starcoder", "tokt": TOKENIZER_TYPE.BPE, "repo": "https://huggingface.co/bigcode/starcoder2-3b", },
|
||||
{ "name": "gpt-2", "tokt": TOKENIZER_TYPE.BPE, "repo": "https://huggingface.co/openai-community/gpt2", },
|
||||
]
|
||||
{"name": "llama-spm", "tokt": TOKENIZER_TYPE.SPM, "repo": "https://huggingface.co/meta-llama/Llama-2-7b-hf", },
|
||||
{"name": "llama-bpe", "tokt": TOKENIZER_TYPE.BPE, "repo": "https://huggingface.co/meta-llama/Meta-Llama-3-8B", },
|
||||
{"name": "phi-3", "tokt": TOKENIZER_TYPE.SPM, "repo": "https://huggingface.co/microsoft/Phi-3-mini-4k-instruct", },
|
||||
{"name": "deepseek-llm", "tokt": TOKENIZER_TYPE.BPE, "repo": "https://huggingface.co/deepseek-ai/deepseek-llm-7b-base", },
|
||||
{"name": "deepseek-coder", "tokt": TOKENIZER_TYPE.BPE, "repo": "https://huggingface.co/deepseek-ai/deepseek-coder-6.7b-base", },
|
||||
{"name": "falcon", "tokt": TOKENIZER_TYPE.BPE, "repo": "https://huggingface.co/tiiuae/falcon-7b", },
|
||||
{"name": "bert-bge", "tokt": TOKENIZER_TYPE.WPM, "repo": "https://huggingface.co/BAAI/bge-small-en-v1.5", },
|
||||
{"name": "mpt", "tokt": TOKENIZER_TYPE.BPE, "repo": "https://huggingface.co/mosaicml/mpt-7b", },
|
||||
{"name": "starcoder", "tokt": TOKENIZER_TYPE.BPE, "repo": "https://huggingface.co/bigcode/starcoder2-3b", },
|
||||
{"name": "gpt-2", "tokt": TOKENIZER_TYPE.BPE, "repo": "https://huggingface.co/openai-community/gpt2", },
|
||||
]
|
||||
|
||||
# make directory "models/tokenizers" if it doesn't exist
|
||||
if not os.path.exists("models/tokenizers"):
|
||||
os.makedirs("models/tokenizers")
|
||||
|
||||
|
||||
def download_file_with_auth(url, token, save_path):
|
||||
headers = {"Authorization": f"Bearer {token}"}
|
||||
response = requests.get(url, headers=headers)
|
||||
if response.status_code == 200:
|
||||
with open(save_path, 'wb') as f:
|
||||
f.write(response.content)
|
||||
print(f"File {save_path} downloaded successfully")
|
||||
logger.info(f"File {save_path} downloaded successfully")
|
||||
else:
|
||||
print(f"Failed to download file. Status code: {response.status_code}")
|
||||
logger.info(f"Failed to download file. Status code: {response.status_code}")
|
||||
|
||||
|
||||
# download the tokenizer models
|
||||
for model in models:
|
||||
|
|
@ -81,10 +89,10 @@ for model in models:
|
|||
if not os.path.exists(f"models/tokenizers/{name}"):
|
||||
os.makedirs(f"models/tokenizers/{name}")
|
||||
else:
|
||||
print(f"Directory models/tokenizers/{name} already exists - skipping")
|
||||
logger.info(f"Directory models/tokenizers/{name} already exists - skipping")
|
||||
continue
|
||||
|
||||
print(f"Downloading {name} to models/tokenizers/{name}")
|
||||
logger.info(f"Downloading {name} to models/tokenizers/{name}")
|
||||
|
||||
url = f"{repo}/raw/main/config.json"
|
||||
save_path = f"models/tokenizers/{name}/config.json"
|
||||
|
|
@ -115,76 +123,76 @@ for model in models:
|
|||
continue
|
||||
|
||||
# create the tokenizer
|
||||
from transformers import AutoTokenizer
|
||||
tokenizer = AutoTokenizer.from_pretrained(f"models/tokenizers/{name}")
|
||||
|
||||
chktok = tokenizer.encode(chktxt)
|
||||
chkhsh = sha256(str(chktok).encode()).hexdigest()
|
||||
|
||||
print(f"model: {name}")
|
||||
print(f"tokt: {tokt}")
|
||||
print(f"repo: {model['repo']}")
|
||||
print(f"chktok: {chktok}")
|
||||
print(f"chkhsh: {chkhsh}")
|
||||
logger.info(f"model: {name}")
|
||||
logger.info(f"tokt: {tokt}")
|
||||
logger.info(f"repo: {model['repo']}")
|
||||
logger.info(f"chktok: {chktok}")
|
||||
logger.info(f"chkhsh: {chkhsh}")
|
||||
|
||||
# print the "pre_tokenizer" content from the tokenizer.json
|
||||
with open(f"models/tokenizers/{name}/tokenizer.json", "r", encoding="utf-8") as f:
|
||||
cfg = json.load(f)
|
||||
pre_tokenizer = cfg["pre_tokenizer"]
|
||||
print("pre_tokenizer: " + json.dumps(pre_tokenizer, indent=4))
|
||||
logger.info("pre_tokenizer: " + json.dumps(pre_tokenizer, indent=4))
|
||||
|
||||
print(f"\n")
|
||||
logger.info("")
|
||||
|
||||
src_ifs += f" if chkhsh == \"{chkhsh}\":\n"
|
||||
src_ifs += f" # ref: {model['repo']}\n"
|
||||
src_ifs += f" res = \"{name}\"\n"
|
||||
|
||||
src_func = ""
|
||||
src_func += " def get_vocab_base_pre(self, tokenizer) -> str:\n"
|
||||
src_func += " # encoding this string and hashing the resulting tokens would (hopefully) give us a unique identifier that\n"
|
||||
src_func += " # is specific for the BPE pre-tokenizer used by the model\n"
|
||||
src_func += " # we will use this unique identifier to write a \"tokenizer.ggml.pre\" entry in the GGUF file which we can\n"
|
||||
src_func += " # use in llama.cpp to implement the same pre-tokenizer\n"
|
||||
src_func += "\n"
|
||||
src_func += f" chktxt = {repr(chktxt)}\n"
|
||||
src_func += "\n"
|
||||
src_func += " chktok = tokenizer.encode(chktxt)\n"
|
||||
src_func += " chkhsh = sha256(str(chktok).encode()).hexdigest()\n"
|
||||
src_func += "\n"
|
||||
src_func += " print(f\"chktok: {chktok}\")\n"
|
||||
src_func += " print(f\"chkhsh: {chkhsh}\")\n"
|
||||
src_func += "\n"
|
||||
src_func += " res = None\n"
|
||||
src_func += "\n"
|
||||
src_func += " # NOTE: if you get an error here, you need to update the convert-hf-to-gguf-update.py script\n"
|
||||
src_func += " # or pull the latest version of the model from Huggingface\n"
|
||||
src_func += " # don't edit the hashes manually!\n"
|
||||
src_func += f"{src_ifs}\n"
|
||||
src_func += " if res is None:\n"
|
||||
src_func += " print(\"\\n\")\n"
|
||||
src_func += " print(\"**************************************************************************************\")\n"
|
||||
src_func += " print(\"** WARNING: The BPE pre-tokenizer was not recognized!\")\n"
|
||||
src_func += " print(\"** There are 2 possible reasons for this:\")\n"
|
||||
src_func += " print(\"** - the model has not been added to convert-hf-to-gguf-update.py yet\")\n"
|
||||
src_func += " print(\"** - the pre-tokenization config has changed upstream\")\n"
|
||||
src_func += " print(\"** Check your model files and convert-hf-to-gguf-update.py and update them accordingly.\")\n"
|
||||
src_func += " print(\"** ref: https://github.com/ggerganov/llama.cpp/pull/6920\")\n"
|
||||
src_func += " print(\"**\")\n"
|
||||
src_func += " print(f\"** chkhsh: {chkhsh}\")\n"
|
||||
src_func += " print(\"**************************************************************************************\")\n"
|
||||
src_func += " print(\"\\n\")\n"
|
||||
src_func += " raise NotImplementedError(\"BPE pre-tokenizer was not recognized - update get_vocab_base_pre()\")\n"
|
||||
src_func += "\n"
|
||||
src_func += " print(f\"tokenizer.ggml.pre: {res}\")\n"
|
||||
src_func += " print(f\"chkhsh: {chkhsh}\")\n"
|
||||
src_func += "\n"
|
||||
src_func += " return res\n"
|
||||
src_func = f"""
|
||||
def get_vocab_base_pre(self, tokenizer) -> str:
|
||||
# encoding this string and hashing the resulting tokens would (hopefully) give us a unique identifier that
|
||||
# is specific for the BPE pre-tokenizer used by the model
|
||||
# we will use this unique identifier to write a "tokenizer.ggml.pre" entry in the GGUF file which we can
|
||||
# use in llama.cpp to implement the same pre-tokenizer
|
||||
|
||||
print(src_func)
|
||||
chktxt = {repr(chktxt)}
|
||||
|
||||
print("\n")
|
||||
print("!!! Copy-paste the function above into convert-hf-to-gguf.py !!!")
|
||||
print("\n")
|
||||
chktok = tokenizer.encode(chktxt)
|
||||
chkhsh = sha256(str(chktok).encode()).hexdigest()
|
||||
|
||||
print(f"chktok: {{chktok}}")
|
||||
print(f"chkhsh: {{chkhsh}}")
|
||||
|
||||
res = None
|
||||
|
||||
# NOTE: if you get an error here, you need to update the convert-hf-to-gguf-update.py script
|
||||
# or pull the latest version of the model from Huggingface
|
||||
# don't edit the hashes manually!
|
||||
{src_ifs}
|
||||
if res is None:
|
||||
print("\\n")
|
||||
print("**************************************************************************************")
|
||||
print("** WARNING: The BPE pre-tokenizer was not recognized!")
|
||||
print("** There are 2 possible reasons for this:")
|
||||
print("** - the model has not been added to convert-hf-to-gguf-update.py yet")
|
||||
print("** - the pre-tokenization config has changed upstream")
|
||||
print("** Check your model files and convert-hf-to-gguf-update.py and update them accordingly.")
|
||||
print("** ref: https://github.com/ggerganov/llama.cpp/pull/6920")
|
||||
print("**")
|
||||
print(f"** chkhsh: {{chkhsh}}")
|
||||
print("**************************************************************************************")
|
||||
print("\\n")
|
||||
raise NotImplementedError("BPE pre-tokenizer was not recognized - update get_vocab_base_pre()")
|
||||
|
||||
print(f"tokenizer.ggml.pre: {{repr(res)}}")
|
||||
print(f"chkhsh: {{chkhsh}}")
|
||||
|
||||
return res
|
||||
"""
|
||||
|
||||
print(src_func) # noqa: NP100
|
||||
|
||||
logger.info("\n")
|
||||
logger.info("!!! Copy-paste the function above into convert-hf-to-gguf.py !!!")
|
||||
logger.info("\n")
|
||||
|
||||
# generate tests for each tokenizer model
|
||||
|
||||
|
|
@ -250,7 +258,6 @@ for model in models:
|
|||
tokt = model["tokt"]
|
||||
|
||||
# create the tokenizer
|
||||
from transformers import AutoTokenizer
|
||||
tokenizer = AutoTokenizer.from_pretrained(f"models/tokenizers/{name}")
|
||||
|
||||
with open(f"models/ggml-vocab-{name}.gguf.inp", "w", encoding="utf-8") as f:
|
||||
|
|
@ -265,15 +272,15 @@ for model in models:
|
|||
f.write(f" {r}")
|
||||
f.write("\n")
|
||||
|
||||
print(f"Tests for {name} written in ./models/ggml-vocab-{name}.gguf.*")
|
||||
logger.info(f"Tests for {name} written in ./models/ggml-vocab-{name}.gguf.*")
|
||||
|
||||
# generate commands for creating vocab files
|
||||
|
||||
print("\nRun the following commands to generate the vocab files for testing:\n")
|
||||
logger.info("\nRun the following commands to generate the vocab files for testing:\n")
|
||||
|
||||
for model in models:
|
||||
name = model["name"]
|
||||
|
||||
print(f"python3 convert-hf-to-gguf.py models/tokenizers/{name}/ --outfile models/ggml-vocab-{name}.gguf --vocab-only")
|
||||
logger.info(f"python3 convert-hf-to-gguf.py models/tokenizers/{name}/ --outfile models/ggml-vocab-{name}.gguf --vocab-only")
|
||||
|
||||
print("\n")
|
||||
logger.info("\n")
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue