fix load_params
This commit is contained in:
parent
2854d26646
commit
a43eb826eb
1 changed files with 2 additions and 15 deletions
|
@ -1791,23 +1791,10 @@ class QwenModel(Model):
|
||||||
class Qwen2Model(Model):
|
class Qwen2Model(Model):
|
||||||
model_arch = gguf.MODEL_ARCH.QWEN2
|
model_arch = gguf.MODEL_ARCH.QWEN2
|
||||||
|
|
||||||
def __init__(self, dir_model: Path, ftype: int, fname_out: Path, is_big_endian: bool, use_temp_file: bool):
|
|
||||||
super().__init__(dir_model, ftype, fname_out, is_big_endian, use_temp_file)
|
|
||||||
self.hparams = Qwen2Model.load_hparams(dir_model)
|
|
||||||
|
|
||||||
@staticmethod
|
|
||||||
def load_hparams(dir_model):
|
|
||||||
with open(dir_model / "config.json", "r", encoding="utf-8") as f1, \
|
|
||||||
open(dir_model / "tokenizer_config.json", "r", encoding="utf-8") as f2:
|
|
||||||
hparams = json.load(f1)
|
|
||||||
hparams.update(json.load(f2))
|
|
||||||
|
|
||||||
return hparams
|
|
||||||
|
|
||||||
def set_vocab(self):
|
def set_vocab(self):
|
||||||
if self.hparams.get("tokenizer_class") == "PreTrainedTokenizerFast":
|
try:
|
||||||
self._set_vocab_sentencepiece()
|
self._set_vocab_sentencepiece()
|
||||||
else:
|
except FileNotFoundError:
|
||||||
self._set_vocab_gpt2()
|
self._set_vocab_gpt2()
|
||||||
|
|
||||||
|
|
||||||
|
|
Loading…
Add table
Add a link
Reference in a new issue