From e71544231c53e679fb7471129b22f783a8a159c1 Mon Sep 17 00:00:00 2001 From: wonjun Jang Date: Sun, 29 Oct 2023 18:29:38 +0000 Subject: [PATCH] Update convert.py --- convert.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/convert.py b/convert.py index 6a162c4d6..77a04062d 100755 --- a/convert.py +++ b/convert.py @@ -1159,7 +1159,7 @@ def load_vocab(path: Path, vocabtype: str | None) -> Vocab: hf_path = vocab_check_and_append_path(path, vocab_file) find_candidates.append(vocab_file) - if vocabtype == "spm" and hf_path is not None: + if hf_path is not None: # A case where there is no tokenizer.model but there is a tokenizer.json and it needs to be loaded into HFVocab. vocabtype = "hf" else: