diff --git a/.github/workflows/python-lint.yml b/.github/workflows/python-lint.yml index c07b9aa83..f4ae65495 100644 --- a/.github/workflows/python-lint.yml +++ b/.github/workflows/python-lint.yml @@ -20,5 +20,5 @@ jobs: - name: flake8 Lint uses: py-actions/flake8@v2 with: - ignore: "E203,E211,E221,E222,E225,E231,E241,E251,E261,E266,E501,E701,E704,W503" + ignore: "E203,E211,E221,E225,E231,E241,E251,E261,E266,E501,E701,E704,W503" exclude: "examples/*,examples/*/**,*/**/__init__.py" diff --git a/convert-hf-to-gguf-update.py b/convert-hf-to-gguf-update.py new file mode 100644 index 000000000..c1d53aaf0 --- /dev/null +++ b/convert-hf-to-gguf-update.py @@ -0,0 +1,162 @@ +# Instructions: +# +# - Add a new model to the "models" list +# - Run the script with your huggingface token: +# +# python3 convert-hf-to-gguf-update.py +# +# - Copy-paste the generated get_vocab_base_pre() function into convert-hf-to-gguf.py +# +# TODO: generate tokenizer tests for llama.cpp +# + +import os +import requests +import sys +import json + +from hashlib import sha256 +from enum import IntEnum, auto + +class TOKENIZER_TYPE(IntEnum): + SPM = auto() + BPE = auto() + WPM = auto() + +# TODO: this string has to exercise as much pre-tokenizer functionality as possible +# will be updated with time - contributions welcome +chktxt = '\n \n\n \n\n\n \t \t\t \t\n \n \n \n \n🚀 (normal) 😶‍🌫️ (multiple emojis concatenated) ✅ 🦙🦙 3 33 333 3333 33333 333333 3333333 33333333 3.3 3..3 3...3 កាន់តែពិសេសអាច😁 ?我想在apple工作1314151天～ ------======= нещо на Български what\'s \'\'\'\'\'\'```````\"\"\"\"......!!!!!!??????' + +if len(sys.argv) == 2: + token = sys.argv[1] +else: + print("Usage: python convert-hf-to-gguf-update.py ") + sys.exit(1) + +# TODO: add models here +models = [ + { "name": "llama-v2", "tokenizer_type": TOKENIZER_TYPE.SPM, "repo": "https://huggingface.co/meta-llama/Llama-2-7b-hf", }, + { "name": "llama-v3", "tokenizer_type": TOKENIZER_TYPE.BPE, "repo": "https://huggingface.co/meta-llama/Meta-Llama-3-8B", }, + { "name": "deepseek-llm", "tokenizer_type": TOKENIZER_TYPE.BPE, "repo": "https://huggingface.co/deepseek-ai/deepseek-llm-7b-chat", }, + { "name": "deepseek-coder", "tokenizer_type": TOKENIZER_TYPE.BPE, "repo": "https://huggingface.co/deepseek-ai/deepseek-coder-6.7b-base", }, + { "name": "bert-bge", "tokenizer_type": TOKENIZER_TYPE.WPM, "repo": "https://huggingface.co/BAAI/bge-small-en-v1.5", }, + ] + +# make directory "models/tokenizers" if it doesn't exist +if not os.path.exists("models/tokenizers"): + os.makedirs("models/tokenizers") + +def download_file_with_auth(url, token, save_path): + headers = {"Authorization": f"Bearer {token}"} + response = requests.get(url, headers=headers) + if response.status_code == 200: + with open(save_path, 'wb') as f: + f.write(response.content) + print("File downloaded successfully.") + else: + print(f"Failed to download file. Status code: {response.status_code}") + +for model in models: + name = model["name"] + repo = model["repo"] + tokenizer_type = model["tokenizer_type"] + + if not os.path.exists(f"models/tokenizers/{name}"): + os.makedirs(f"models/tokenizers/{name}") + else: + print(f"Directory models/tokenizers/{name} already exists - skipping") + continue + + print(f"Downloading {name} to models/tokenizers/{name}") + + url = f"{repo}/raw/main/tokenizer.json" + save_path = f"models/tokenizers/{name}/tokenizer.json" + download_file_with_auth(url, token, save_path) + + if tokenizer_type == TOKENIZER_TYPE.SPM: + url = f"{repo}/resolve/main/tokenizer.model" + save_path = f"models/tokenizers/{name}/tokenizer.model" + download_file_with_auth(url, token, save_path) + + url = f"{repo}/raw/main/tokenizer_config.json" + save_path = f"models/tokenizers/{name}/tokenizer_config.json" + download_file_with_auth(url, token, save_path) + +# generate the source code for the convert-hf-to-gguf.py:get_vocab_base_pre() function: +# TODO: auto-update convert-hf-to-gguf.py with the generated function + +src_ifs = "" +for model in models: + name = model["name"] + tokenizer_type = model["tokenizer_type"] + + if tokenizer_type == TOKENIZER_TYPE.SPM: + continue + + # create the tokenizer + from transformers import AutoTokenizer + tokenizer = AutoTokenizer.from_pretrained(f"models/tokenizers/{name}") + + chktok = tokenizer.encode(chktxt) + chkhsh = sha256(str(chktok).encode()).hexdigest() + + print(f"model: {name}") + print(f"tokenizer_type: {tokenizer_type}") + print(f"repo: {model['repo']}") + print(f"chktok: {chktok}") + print(f"chkhsh: {chkhsh}") + + # print the "pre_tokenizer" content from the tokenizer.json + with open(f"models/tokenizers/{name}/tokenizer.json", "r") as f: + cfg = json.load(f) + pre_tokenizer = cfg["pre_tokenizer"] + print("pre_tokenizer: " + json.dumps(pre_tokenizer, indent=4)) + + print(f"\n") + + src_ifs += f" if chkhsh == \"{chkhsh}\":\n" + src_ifs += f" # ref: {model['repo']}\n" + src_ifs += f" res = \"{name}\"\n" + +src_func = "" +src_func += " def get_vocab_base_pre(self, tokenizer) -> str:\n" +src_func += " # encoding this string and hashing the resulting tokens would (hopefully) give us a unique identifier that\n" +src_func += " # is specific for the BPE pre-tokenizer used by the model\n" +src_func += " # we will use this unique identifier to write a \"tokenizer.ggml.pre\" entry in the GGUF file which we can\n" +src_func += " # use in llama.cpp to implement the same pre-tokenizer\n" +src_func += "\n" +src_func += f" chktxt = {repr(chktxt)}\n" +src_func += "\n" +src_func += " chktok = tokenizer.encode(chktxt)\n" +src_func += " chkhsh = sha256(str(chktok).encode()).hexdigest()\n" +src_func += "\n" +src_func += " print(f\"chktok: {chktok}\")\n" +src_func += " print(f\"chkhsh: {chkhsh}\")\n" +src_func += "\n" +src_func += " res = None\n" +src_func += "\n" +src_func += " # NOTE: if you get an error here, you need to add the model to the if-elif chain below\n" +src_func += f"{src_ifs}\n" +src_func += " if res is None:\n" +src_func += " print(f\"\\n\")\n" +src_func += " print(f\"**************************************************************************************\")\n" +src_func += " print(f\"** WARNING: The BPE pre-tokenizer was not recognized!\")\n" +src_func += " print(f\"** This means that it was not added yet or you are using an older version.\")\n" +src_func += " print(f\"** Check convert-hf-to-gguf-update.py and update it accordingly.\")\n" +src_func += " print(f\"**\")\n" +src_func += " print(f\"** chkhsh: {chkhsh}\")\n" +src_func += " print(f\"**************************************************************************************\")\n" +src_func += " print(f\"\\n\")\n" +src_func += " raise NotImplementedError(\"BPE pre-tokenizer was not recognized - update get_vocab_base_pre()\")\n" +src_func += "\n" +src_func += " print(f\"tokenizer.ggml.pre: {res}\")\n" +src_func += " print(f\"chkhsh: {chkhsh}\")\n" +src_func += "\n" +src_func += " return res\n" + +print(src_func) + +print("\n") +print("!!! Copy-paste the function above into convert-hf-to-gguf.py !!!") +print("\n") + diff --git a/convert-hf-to-gguf.py b/convert-hf-to-gguf.py index af698a23a..9b2f68cfd 100755 --- a/convert-hf-to-gguf.py +++ b/convert-hf-to-gguf.py @@ -11,6 +11,7 @@ import sys from abc import ABC, abstractmethod from enum import IntEnum from pathlib import Path +from hashlib import sha256 from typing import TYPE_CHECKING, Any, Callable, ContextManager, Iterator, Sequence, TypeVar, cast import numpy as np @@ -376,16 +377,19 @@ class Model(ABC): return tokens, toktypes, tokpre + # NOTE: this function is generated by convert-hf-to-gguf-update.py + # do not modify it manually! + # ref: https://github.com/ggerganov/llama.cpp/pull/6920 def get_vocab_base_pre(self, tokenizer) -> str: # encoding this string and hashing the resulting tokens would (hopefully) give us a unique identifier that # is specific for the BPE pre-tokenizer used by the model # we will use this unique identifier to write a "tokenizer.ggml.pre" entry in the GGUF file which we can # use in llama.cpp to implement the same pre-tokenizer - chktxt = "\n \n\n \n\n\n \t \t\t \t\n \n \n \n \n🚀 (normal) 😶‍🌫️ (multiple emojis concatenated) ✅ 🦙🦙 3 33 333 3333 33333 333333 3333333 33333333 3.3 3..3 3...3 កាន់តែពិសេសអាច😁 ?我想在apple工作1314151天～ ------======= нещо на Български what's ''''''```````\"\"\"\"......!!!!!!??????" + chktxt = '\n \n\n \n\n\n \t \t\t \t\n \n \n \n \n🚀 (normal) 😶\u200d🌫️ (multiple emojis concatenated) ✅ 🦙🦙 3 33 333 3333 33333 333333 3333333 33333333 3.3 3..3 3...3 កាន់តែពិសេសអាច😁 ?我想在apple工作1314151天～ ------======= нещо на Български what\'s \'\'\'\'\'\'```````""""......!!!!!!??????' chktok = tokenizer.encode(chktxt) - chkhsh = hash(tuple(chktok)) + chkhsh = sha256(str(chktok).encode()).hexdigest() print(f"chktok: {chktok}") print(f"chkhsh: {chkhsh}") @@ -393,21 +397,34 @@ class Model(ABC): res = None # NOTE: if you get an error here, you need to add the model to the if-elif chain below - # observe the stdout for the chkhsh value and add it to the chain - if self.model_arch == gguf.MODEL_ARCH.LLAMA: - if chkhsh == -3290901550109860290: - # ref: https://huggingface.co/meta-llama/Meta-Llama-3-8B-Instruct/blob/main/tokenizer.json - res = "llama3" - if chkhsh == 5332289095291046364: - # ref: https://huggingface.co/deepseek-ai/deepseek-llm-7b-chat/blob/main/tokenizer.json - res = "deepseek-llm" - if chkhsh == 4190561703949727616: - # ref: https://huggingface.co/deepseek-ai/deepseek-coder-6.7b-instruct/blob/main/tokenizer.json - res = "deepseek-coder" + if chkhsh == "0fc850edd52197e357970116fbf58f6c2567f259cdc1bfc3df081d7e4bc658c1": + # ref: https://huggingface.co/meta-llama/Meta-Llama-3-8B + res = "llama-v3" + if chkhsh == "58c3d0e812ae7fa6a20931006d2398274732c105a9a964c148c43cf898c5fb7a": + # ref: https://huggingface.co/deepseek-ai/deepseek-llm-7b-chat + res = "deepseek-llm" + if chkhsh == "0438d2a948d7fb26c7a662705ac68374f3138ee29e44f133b1f059203500fb4d": + # ref: https://huggingface.co/deepseek-ai/deepseek-coder-6.7b-base + res = "deepseek-coder" + if chkhsh == "406f3f61e1c67d7b0456c5df2fce5cbb30c77dd3671a436b07a6c510303f721e": + # ref: https://huggingface.co/BAAI/bge-small-en-v1.5 + res = "bert-bge" if res is None: + print(f"\n") + print(f"**************************************************************************************") + print(f"** WARNING: The BPE pre-tokenizer was not recognized!") + print(f"** This means that it was not added yet or you are using an older version.") + print(f"** Check convert-hf-to-gguf-update.py and update it accordingly.") + print(f"**") + print(f"** chkhsh: {chkhsh}") + print(f"**************************************************************************************") + print(f"\n") raise NotImplementedError("BPE pre-tokenizer was not recognized - update get_vocab_base_pre()") + print(f"tokenizer.ggml.pre: {res}") + print(f"chkhsh: {chkhsh}") + return res def _set_vocab_gpt2(self) -> None: diff --git a/llama.cpp b/llama.cpp index 73c31382e..eaf9a8da0 100644 --- a/llama.cpp +++ b/llama.cpp @@ -4330,19 +4330,29 @@ static void llm_load_vocab( vocab.special_mask_id = -1; } - if (tokenizer_pre.empty()) { - LLAMA_LOG_WARN("%s: missing pre-tokenizer type, using: 'default'\n", __func__); - vocab.type_pre = LLAMA_VOCAB_PRE_TYPE_DEFAULT; - } else if (tokenizer_pre == "default") { - vocab.type_pre = LLAMA_VOCAB_PRE_TYPE_DEFAULT; - } else if (tokenizer_pre == "llama3") { - vocab.type_pre = LLAMA_VOCAB_PRE_TYPE_LLAMA3; - } else if (tokenizer_pre == "deepseek-llm") { - vocab.type_pre = LLAMA_VOCAB_PRE_TYPE_DEEPSEEK_LLM; - } else if (tokenizer_pre == "deepseek-coder") { - vocab.type_pre = LLAMA_VOCAB_PRE_TYPE_DEEPSEEK_CODER; + // for now, only BPE models have pre-tokenizers + if (vocab.type == LLAMA_VOCAB_TYPE_BPE) { + if (tokenizer_pre.empty()) { + LLAMA_LOG_WARN("%s: missing pre-tokenizer type, using: 'default'\n", __func__); + vocab.type_pre = LLAMA_VOCAB_PRE_TYPE_DEFAULT; + } else if ( + tokenizer_pre == "default") { + vocab.type_pre = LLAMA_VOCAB_PRE_TYPE_DEFAULT; + } else if ( + tokenizer_pre == "llama3" || + tokenizer_pre == "llama-v3") { + vocab.type_pre = LLAMA_VOCAB_PRE_TYPE_LLAMA3; + } else if ( + tokenizer_pre == "deepseek-llm") { + vocab.type_pre = LLAMA_VOCAB_PRE_TYPE_DEEPSEEK_LLM; + } else if ( + tokenizer_pre == "deepseek-coder") { + vocab.type_pre = LLAMA_VOCAB_PRE_TYPE_DEEPSEEK_CODER; + } else { + throw std::runtime_error(format("unknown pre-tokenizer type: '%s'", tokenizer_pre.c_str())); + } } else { - throw std::runtime_error(format("unknown pre-tokenizer type: '%s'", tokenizer_pre.c_str())); + vocab.type_pre = LLAMA_VOCAB_PRE_TYPE_DEFAULT; } } diff --git a/models/ggml-vocab-llama-v3.gguf b/models/ggml-vocab-llama-v3.gguf index 8b97afbf5..bf7354c26 100644 Binary files a/models/ggml-vocab-llama-v3.gguf and b/models/ggml-vocab-llama-v3.gguf differ