From a43eb826eb0982d858feaac2c22cd10aa4bd6665 Mon Sep 17 00:00:00 2001 From: JustinLin610 Date: Fri, 19 Apr 2024 17:05:51 +0800 Subject: [PATCH] fix load_params --- convert-hf-to-gguf.py | 17 ++--------------- 1 file changed, 2 insertions(+), 15 deletions(-) diff --git a/convert-hf-to-gguf.py b/convert-hf-to-gguf.py index 2a99a0588..8912c97d5 100755 --- a/convert-hf-to-gguf.py +++ b/convert-hf-to-gguf.py @@ -1790,24 +1790,11 @@ class QwenModel(Model): @Model.register("Qwen2ForCausalLM") class Qwen2Model(Model): model_arch = gguf.MODEL_ARCH.QWEN2 - - def __init__(self, dir_model: Path, ftype: int, fname_out: Path, is_big_endian: bool, use_temp_file: bool): - super().__init__(dir_model, ftype, fname_out, is_big_endian, use_temp_file) - self.hparams = Qwen2Model.load_hparams(dir_model) - - @staticmethod - def load_hparams(dir_model): - with open(dir_model / "config.json", "r", encoding="utf-8") as f1, \ - open(dir_model / "tokenizer_config.json", "r", encoding="utf-8") as f2: - hparams = json.load(f1) - hparams.update(json.load(f2)) - - return hparams def set_vocab(self): - if self.hparams.get("tokenizer_class") == "PreTrainedTokenizerFast": + try: self._set_vocab_sentencepiece() - else: + except FileNotFoundError: self._set_vocab_gpt2()