Compare commits

...
Sign in to create a new pull request.

1 commit

Author SHA1 Message Date
Jared Van Bortel
b0547d2196 gguf-py : fail fast on nonsensical special token IDs 2023-12-15 18:06:42 -05:00

View file

@ -109,8 +109,10 @@ class SpecialVocab:
return True
def _set_special_token(self, typ: str, tid: Any) -> None:
if not isinstance(tid, int) or tid < 0:
if not isinstance(tid, int):
return
if tid < 0:
raise ValueError(f'invalid value for special token type {typ}: {tid}')
if self.n_vocab is None or tid < self.n_vocab:
if typ in self.special_token_ids:
return