diff --git a/convert.py b/convert.py index 955b6546f..4077be7f2 100755 --- a/convert.py +++ b/convert.py @@ -414,6 +414,7 @@ class SentencePieceVocab: def __repr__(self) -> str: return f"" + class HFVocab: def __init__(self, fname_tokenizer: Path, fname_added_tokens: Path | None) -> None: self.tokenizer = AutoTokenizer.from_pretrained(str(fname_tokenizer))