diff --git a/convert.py b/convert.py index 5d2f8aa55..ee7148a65 100644 --- a/convert.py +++ b/convert.py @@ -217,13 +217,13 @@ class XgenVocab: def all_tokens(self) -> Iterable[Tuple[bytes, float]]: for index in range(0, self.vocab_size_base): - token = self.xt._convert_id_to_token(index) + token = self.xt.encoder.decode_single_token_bytes(index) yield (token, float(index)) for index in range(self.vocab_size_base, self.vocab_size): yield (b'', float(index)) def __repr__(self) -> str: - return f"" + return f"" class SentencePieceVocab: