From 822127e6d1daaa09f3336579cc2b51f5211d8af6 Mon Sep 17 00:00:00 2001 From: KerfuffleV2 Date: Thu, 31 Aug 2023 11:24:46 -0600 Subject: [PATCH] Remove unnecessary conditional in addl token error handling --- convert.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/convert.py b/convert.py index 03653b023..5b547f068 100755 --- a/convert.py +++ b/convert.py @@ -339,8 +339,7 @@ class BpeVocab: actual_ids = sorted(added_tokens.values()) if expected_ids != actual_ids: expected_end_id = vocab_size + len(actual_ids) - 1 - if actual_ids[0] != vocab_size or actual_ids[-1] != expected_end_id: - raise Exception(f"Expected the {len(actual_ids)} added token ID(s) to be sequential in the range {vocab_size} - {expected_end_id}; got {actual_ids}") + raise Exception(f"Expected the {len(actual_ids)} added token ID(s) to be sequential in the range {vocab_size} - {expected_end_id}; got {actual_ids}") items = sorted(added_tokens.items(), key=lambda text_idx: text_idx[1]) self.added_tokens_list = [text for (text, idx) in items]