vbatts/llama.cpp
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Update convert-hf-to-gguf.py

Browse source 
Replacing with a less risky alternative to the 'assert' statement
This commit is contained in:
0xspringtime 2024-06-19 14:12:23 -04:00 • committed by GitHub
parent 9c77ec1d74
commit 620633e9d8
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
1 changed files with 3 additions and 1 deletions
Download patch file Download diff file Expand all files Collapse all files

4
convert-hf-to-gguf.py
View file

@ -967,7 +967,9 @@ class XverseModel(Model):
from transformers import AutoTokenizer
tokenizer = AutoTokenizer.from_pretrained(dir_model)
vocab_size = hparams.get("vocab_size", len(tokenizer.vocab))
assert max(tokenizer.vocab.values()) < vocab_size
if max(tokenizer.get_vocab().values()) >= vocab_size:
raise ValueError("Vocabulary size exceeds expected maximum size.")
reverse_vocab: dict[int, str] = {id_: encoded_tok for encoded_tok, id_ in tokenizer.vocab.items()}
added_vocab = tokenizer.get_added_vocab()