vbatts/llama.cpp
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Remove unnecessary conditional in addl token error handling

Browse source
This commit is contained in:
KerfuffleV2 2023-08-31 11:24:46 -06:00
parent 25c0d79b0f
commit 822127e6d1
1 changed files with 1 additions and 2 deletions
Download patch file Download diff file Expand all files Collapse all files

3
convert.py
View file

@ -339,8 +339,7 @@ class BpeVocab:
actual_ids = sorted(added_tokens.values())
if expected_ids != actual_ids:
expected_end_id = vocab_size + len(actual_ids) - 1
if actual_ids[0] != vocab_size or actual_ids[-1] != expected_end_id:
raise Exception(f"Expected the {len(actual_ids)} added token ID(s) to be sequential in the range {vocab_size} - {expected_end_id}; got {actual_ids}")
raise Exception(f"Expected the {len(actual_ids)} added token ID(s) to be sequential in the range {vocab_size} - {expected_end_id}; got {actual_ids}")
items = sorted(added_tokens.items(), key=lambda text_idx: text_idx[1])
self.added_tokens_list = [text for (text, idx) in items]