From 3937100adb6df051011d41d4af3e697fe232547c Mon Sep 17 00:00:00 2001 From: Pierrick HYMBERT Date: Sat, 6 Apr 2024 15:57:57 +0200 Subject: [PATCH] model: dbrx, trust remote code --- convert-hf-to-gguf.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/convert-hf-to-gguf.py b/convert-hf-to-gguf.py index 4b9b0fc8e..a847d118d 100755 --- a/convert-hf-to-gguf.py +++ b/convert-hf-to-gguf.py @@ -1451,7 +1451,7 @@ class Qwen2MoeModel(Model): toktypes: list[int] = [] from transformers import AutoTokenizer - tokenizer = AutoTokenizer.from_pretrained(dir_model) + tokenizer = AutoTokenizer.from_pretrained(dir_model, trust_remote_code=True) vocab_size = tokenizer.vocab_size reverse_vocab = {id_: encoded_tok for encoded_tok, id_ in tokenizer.get_vocab().items()}