vbatts/llama.cpp
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

convert : use non-fast T5 tokenizer

Browse source
This commit is contained in:
Georgi Gerganov 2024-07-02 19:29:26 +03:00
parent 17bb0eaec3
commit 703764a382
No known key found for this signature in database
GPG key ID: 449E073F9DC10735
1 changed files with 8 additions and 2 deletions
Download patch file Download diff file Expand all files Collapse all files

6
convert-hf-to-gguf-update.py
View file

@ -147,6 +147,9 @@ for model in models:
# create the tokenizer # create the tokenizer
try: try:
if name == "t5":
tokenizer = AutoTokenizer.from_pretrained(f"models/tokenizers/{name}", use_fast=False)
else:
tokenizer = AutoTokenizer.from_pretrained(f"models/tokenizers/{name}") tokenizer = AutoTokenizer.from_pretrained(f"models/tokenizers/{name}")
except OSError as e: except OSError as e:
logger.error(f"Error loading tokenizer for model {name}. The model may not exist or is not accessible with the provided token. Error: {e}") logger.error(f"Error loading tokenizer for model {name}. The model may not exist or is not accessible with the provided token. Error: {e}")
@ -306,6 +309,9 @@ for model in models:
# create the tokenizer # create the tokenizer
try: try:
if name == "t5":
tokenizer = AutoTokenizer.from_pretrained(f"models/tokenizers/{name}", use_fast=False)
else:
tokenizer = AutoTokenizer.from_pretrained(f"models/tokenizers/{name}") tokenizer = AutoTokenizer.from_pretrained(f"models/tokenizers/{name}")
except OSError as e: except OSError as e:
logger.error(f"Failed to load tokenizer for model {name}. Error: {e}") logger.error(f"Failed to load tokenizer for model {name}. Error: {e}")