vbatts/llama.cpp
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Add newline between classes

Browse source
This commit is contained in:
wonjun Jang 2023-10-15 10:37:41 +00:00 • committed by GitHub
parent 1f16e5f234
commit e876aec189
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
1 changed files with 1 additions and 0 deletions
Download patch file Download diff file Expand all files Collapse all files

1
convert.py
View file

@ -414,6 +414,7 @@ class SentencePieceVocab:
def __repr__(self) -> str:
return f"<SentencePieceVocab with {self.vocab_size_base} base tokens and {len(self.added_tokens_list)} added tokens>"
class HFVocab:
def __init__(self, fname_tokenizer: Path, fname_added_tokens: Path | None) -> None:
self.tokenizer = AutoTokenizer.from_pretrained(str(fname_tokenizer))