diff --git a/convert-hf-to-gguf-update.py b/convert-hf-to-gguf-update.py index 7e0a808fa..fd0fd9bf3 100644 --- a/convert-hf-to-gguf-update.py +++ b/convert-hf-to-gguf-update.py @@ -1,3 +1,12 @@ +# This script downloads the tokenizer models of the specified models from Huggingface and +# generates the get_vocab_base_pre() function for convert-hf-to-gguf.py +# +# This is necessary in order to analyze the type of pre-tokenizer used by the model and +# provide the necessary information to llama.cpp via the GGUF header in order to implement +# the same pre-tokenizer. +# +# ref: https://github.com/ggerganov/llama.cpp/pull/6920 +# # Instructions: # # - Add a new model to the "models" list @@ -9,6 +18,7 @@ # - Update llama.cpp with the new pre-tokenizer if necessary # # TODO: generate tokenizer tests for llama.cpp +# TODO: automate the update of convert-hf-to-gguf.py # import os @@ -138,6 +148,7 @@ src_func += "\n" src_func += " res = None\n" src_func += "\n" src_func += " # NOTE: if you get an error here, you need to add the model to the if-elif chain below\n" +src_func += " # don't do this manually - use the convert-hf-to-gguf-update.py script!\n" src_func += f"{src_ifs}\n" src_func += " if res is None:\n" src_func += " print( \"\\n\")\n" diff --git a/convert-hf-to-gguf.py b/convert-hf-to-gguf.py index 40fb53c8a..6bf987767 100755 --- a/convert-hf-to-gguf.py +++ b/convert-hf-to-gguf.py @@ -282,6 +282,7 @@ class Model(ABC): res = None # NOTE: if you get an error here, you need to add the model to the if-elif chain below + # don't do this manually - use the convert-hf-to-gguf-update.py script! if chkhsh == "0fc850edd52197e357970116fbf58f6c2567f259cdc1bfc3df081d7e4bc658c1": # ref: https://huggingface.co/meta-llama/Meta-Llama-3-8B res = "llama-v3"