vbatts/llama.cpp
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Update convert.py

Browse source
This commit is contained in:
wonjun Jang 2023-10-29 18:29:38 +00:00 • committed by GitHub
parent 97f690ab51
commit e71544231c
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
1 changed files with 1 additions and 1 deletions
Download patch file Download diff file Expand all files Collapse all files

2
convert.py
View file

@ -1159,7 +1159,7 @@ def load_vocab(path: Path, vocabtype: str | None) -> Vocab:
hf_path = vocab_check_and_append_path(path, vocab_file)
find_candidates.append(vocab_file)
if vocabtype == "spm" and hf_path is not None:
if hf_path is not None:
# A case where there is no tokenizer.model but there is a tokenizer.json and it needs to be loaded into HFVocab.
vocabtype = "hf"
else: