From c7b636e9507a9ae7dd81242004ab1c9497ea942a Mon Sep 17 00:00:00 2001 From: wonjun Jang Date: Sun, 15 Oct 2023 09:43:22 +0000 Subject: [PATCH] change add_meta_vocab fucntion --- convert.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/convert.py b/convert.py index 98ea6d9c8..e84db460f 100755 --- a/convert.py +++ b/convert.py @@ -913,12 +913,12 @@ class OutputFile: scores.append(score) toktypes.append(toktype) - if isinstance(vocab, SentencePieceVocab): + if isinstance(vocab, SentencePieceVocab) or isinstance(vocab, HFVocab): self.gguf.add_tokenizer_model("llama") elif isinstance(vocab, BpeVocab): self.gguf.add_tokenizer_model("gpt2") else: - raise ValueError(f'Unknown vocab type: Not BpeVocab or SentencePieceVocab') + raise ValueError(f'Unknown vocab type: Not BpeVocab or SentencePieceVocab or HFVocab') self.gguf.add_token_list(tokens) self.gguf.add_token_scores(scores) self.gguf.add_token_types(toktypes)