vbatts/llama.cpp
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Get rope scale from HF models

Browse source
This commit is contained in:
Nigel Bosch 2023-08-24 15:09:22 -05:00
parent d0f77b1353
commit 8d0d83b391
1 changed files with 22 additions and 16 deletions
Download patch file Download diff file Expand all files Collapse all files

38
convert.py
View file

@ -105,6 +105,7 @@ class Params:
f_norm_eps: float f_norm_eps: float
f_rope_freq_base: Optional[float] = None f_rope_freq_base: Optional[float] = None
f_rope_scale: Optional[float] = None
ftype: Optional[GGMLFileType] = None ftype: Optional[GGMLFileType] = None
@ -160,13 +161,14 @@ class Params:
def loadHFTransformerJson(model: 'LazyModel', config_path: 'Path') -> 'Params': def loadHFTransformerJson(model: 'LazyModel', config_path: 'Path') -> 'Params':
config = json.load(open(config_path)) config = json.load(open(config_path))
n_vocab = config["vocab_size"] n_vocab = config["vocab_size"]
n_embd = config["hidden_size"] n_embd = config["hidden_size"]
n_layer = config["num_hidden_layers"] n_layer = config["num_hidden_layers"]
n_ff = config["intermediate_size"] n_ff = config["intermediate_size"]
n_head = config["num_attention_heads"] n_head = config["num_attention_heads"]
n_head_kv = config["num_key_value_heads"] if "num_key_value_heads" in config else n_head n_head_kv = config["num_key_value_heads"] if "num_key_value_heads" in config else n_head
f_norm_eps = config["rms_norm_eps"] f_norm_eps = config["rms_norm_eps"]
f_rope_scale = config["rope_scaling"]["factor"] if "rope_scaling" in config and "factor" in config["rope_scaling"] else None
n_mult = Params.find_n_mult(n_ff, n_embd) n_mult = Params.find_n_mult(n_ff, n_embd)
@ -179,15 +181,16 @@ class Params:
"Suggestion: provide 'config.json' of the model in the same directory containing model files.") "Suggestion: provide 'config.json' of the model in the same directory containing model files.")
return Params( return Params(
n_vocab = n_vocab, n_vocab = n_vocab,
n_embd = n_embd, n_embd = n_embd,
n_mult = n_mult, n_mult = n_mult,
n_layer = n_layer, n_layer = n_layer,
n_ctx = n_ctx, n_ctx = n_ctx,
n_ff = n_ff, n_ff = n_ff,
n_head = n_head, n_head = n_head,
n_head_kv = n_head_kv, n_head_kv = n_head_kv,
f_norm_eps = f_norm_eps, f_norm_eps = f_norm_eps,
f_rope_scale = f_rope_scale,
) )
# LLaMA v2 70B params.json # LLaMA v2 70B params.json
@ -771,6 +774,9 @@ class OutputFile:
if params.f_rope_freq_base: if params.f_rope_freq_base:
self.gguf.add_rope_freq_base(params.f_rope_freq_base) self.gguf.add_rope_freq_base(params.f_rope_freq_base)
if params.f_rope_scale:
self.gguf.add_rope_scale_linear(params.f_rope_scale)
if params.ftype: if params.ftype:
self.gguf.add_file_type(params.ftype) self.gguf.add_file_type(params.ftype)