make mypy happy
This commit is contained in:
parent
8fabb0132c
commit
c3b1c12fdd
1 changed files with 7 additions and 10 deletions
17
convert.py
17
convert.py
|
@ -329,7 +329,7 @@ class VocabLoader:
|
|||
self.special_ids: set[int] = set(self.tokenizer.all_special_ids)
|
||||
self.vocab_size_base: int = self.tokenizer.vocab_size
|
||||
self.vocab_size: int = self.vocab_size_base + len(self.added_tokens_dict)
|
||||
self.fname_tokenizer: str = fname_tokenizer
|
||||
self.fname_tokenizer: Path = fname_tokenizer
|
||||
|
||||
vocab_file = "tokenizer.model"
|
||||
path_candidate = find_vocab_file_path(self.fname_tokenizer, vocab_file)
|
||||
|
@ -373,9 +373,8 @@ class VocabLoader:
|
|||
|
||||
def get_token_score(self, token_id: int) -> float:
|
||||
if self.spm is not None and token_id < self.spm.vocab_size():
|
||||
return self.spm.get_score(token_id)
|
||||
else:
|
||||
return 0.0
|
||||
return cast(float, self.spm.get_score(token_id))
|
||||
return 0.0
|
||||
|
||||
def added_tokens(self) -> Iterable[tuple[bytes, float, gguf.TokenType]]:
|
||||
|
||||
|
@ -1110,13 +1109,11 @@ def find_vocab_file_path(path: Path, vocab_file: str) -> Optional[Path]:
|
|||
path3 = path.parent / vocab_file
|
||||
|
||||
if path2.exists():
|
||||
path = path2
|
||||
elif path3.exists():
|
||||
path = path3
|
||||
else:
|
||||
path = None
|
||||
return path2
|
||||
if path3.exists():
|
||||
return path3
|
||||
|
||||
return path
|
||||
return None
|
||||
|
||||
|
||||
def default_outfile(model_paths: list[Path], file_type: GGMLFileType) -> Path:
|
||||
|
|
Loading…
Add table
Add a link
Reference in a new issue