Add error context with 'raise from'

This commit is contained in:
wonjun Jang 2023-10-18 01:30:38 +00:00 committed by GitHub
parent a5b26b660b
commit 5a1f178091
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23

View file

@ -418,11 +418,11 @@ class HFVocab:
def __init__(self, fname_tokenizer: Path, fname_added_tokens: Path | None) -> None: def __init__(self, fname_tokenizer: Path, fname_added_tokens: Path | None) -> None:
try: try:
from transformers import AutoTokenizer from transformers import AutoTokenizer
except ModuleNotFoundError: except ImportError as e:
raise ImportError( raise ImportError(
"To use HFVocab, please install the `transformers` package. " "To use HFVocab, please install the `transformers` package. "
"You can install it with `pip install transformers`." "You can install it with `pip install transformers`."
) ) from e
self.tokenizer = AutoTokenizer.from_pretrained(str(fname_tokenizer)) self.tokenizer = AutoTokenizer.from_pretrained(str(fname_tokenizer))
@ -445,6 +445,7 @@ class HFVocab:
self.vocab_size: int = self.vocab_size_base + len(self.added_tokens_list) self.vocab_size: int = self.vocab_size_base + len(self.added_tokens_list)
self.fname_tokenizer = fname_tokenizer self.fname_tokenizer = fname_tokenizer
self.fname_added_tokens = fname_added_tokens self.fname_added_tokens = fname_added_tokens
def hf_tokens(self) -> Iterable[tuple[bytes, float, gguf.TokenType]]: def hf_tokens(self) -> Iterable[tuple[bytes, float, gguf.TokenType]]:
tokenizer = self.tokenizer tokenizer = self.tokenizer
reverse_vocab = {id: encoded_tok for encoded_tok, id in tokenizer.vocab.items()} reverse_vocab = {id: encoded_tok for encoded_tok, id in tokenizer.vocab.items()}