From ff12b8fbd629bfa55e6b8457827e35a88ff2d755 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=EA=B9=80=EC=8A=B9=EB=8D=95/Infrastructure=EA=B7=B8?= =?UTF-8?q?=EB=A3=B9=28YA=29?= Date: Sun, 15 Oct 2023 21:30:09 +0900 Subject: [PATCH] move back vocab_size where it was --- convert.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/convert.py b/convert.py index e9fe25b78..96b6c6f84 100755 --- a/convert.py +++ b/convert.py @@ -359,14 +359,14 @@ class BpeVocab: class SentencePieceVocab: def __init__(self, fname_tokenizer: Path, fname_added_tokens: Path | None) -> None: self.sentencepiece_tokenizer = SentencePieceProcessor(str(fname_tokenizer)) - vocab_size: int = self.sentencepiece_tokenizer.vocab_size() - added_tokens: dict[str, int] if fname_added_tokens is not None: added_tokens = json.load(open(fname_added_tokens, encoding="utf-8")) else: added_tokens = {} + vocab_size: int = self.sentencepiece_tokenizer.vocab_size() + new_tokens: dict[int, str] = {id: piece for piece, id in added_tokens.items() if id >= vocab_size} expected_new_ids: list[int] = list(range(vocab_size, vocab_size + len(new_tokens))) actual_new_ids: list[int] = sorted(new_tokens.keys())