vbatts/llama.cpp
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

gguf-py: dbrx: reverse again the MOE tensors mapping:

Browse source 
layer.ffn_up_exps   -> Up-projection weights (w1)
    layer.ffn_gate_exps -> Gating weights (v1)
    layer.ffn_down_exps -> Down-projection weights (w2)
This commit is contained in:
Pierrick HYMBERT 2024-04-09 02:41:39 +02:00
parent ac82aa0e63
commit ac75fbd8c5
2 changed files with 4 additions and 4 deletions
Download patch file Download diff file Expand all files Collapse all files

4
convert-hf-to-gguf.py
View file

@ -1527,9 +1527,9 @@ class DbrxModel(Model):
# But llama.cpp moe graph works differently
# AND the dimensions in ggml are typically in the reverse order of the pytorch dimensions
# so (n_expert, n_ff, n_embd) in pytorch is {n_embd, n_ff, n_expert} in ggml_tensor
exp_tensor_names = {"ffn.experts.mlp.w1": None, # LLM_TENSOR_FFN_GATE_EXPS ggml_tensor->ne{n_embd, n_ff, n_expert}
exp_tensor_names = {"ffn.experts.mlp.v1": None, # LLM_TENSOR_FFN_GATE_EXPS ggml_tensor->ne{n_embd, n_ff, n_expert}
"ffn.experts.mlp.w2": (0, 2, 1), # LLM_TENSOR_FFN_DOWN_EXPS ggml_tensor->ne{n_ff, n_embd, n_expert}
"ffn.experts.mlp.v1": None} # LLM_TENSOR_FFN_UP_EXPS ggml_tensor->ne{n_embd, n_ff, n_expert}
"ffn.experts.mlp.w1": None} # LLM_TENSOR_FFN_UP_EXPS ggml_tensor->ne{n_embd, n_ff, n_expert}
experts = False
for exp_tensor_name in exp_tensor_names.keys():
if name.find(exp_tensor_name) != -1 and name.find(".weight") == -1:

4
gguf-py/gguf/tensor_mapping.py
View file

@ -238,7 +238,7 @@ class TensorNameMap:
MODEL_TENSOR.FFN_UP_EXP: (
"layers.{bid}.feed_forward.experts.w3", # mixtral (merged)
"transformer.decoder_layer.{bid}.moe.linear_v", # Grok (merged)
"transformer.blocks.{bid}.ffn.experts.mlp.v1", # dbrx
"transformer.blocks.{bid}.ffn.experts.mlp.w1", # dbrx
),
# AWQ-activation gate
@ -259,7 +259,7 @@ class TensorNameMap:
MODEL_TENSOR.FFN_GATE_EXP: (
"layers.{bid}.feed_forward.experts.w1", # mixtral (merged)
"transformer.decoder_layer.{bid}.moe.linear", # Grok (merged)
"transformer.blocks.{bid}.ffn.experts.mlp.w1", # dbrx
"transformer.blocks.{bid}.ffn.experts.mlp.v1", # dbrx
),
# Feed-forward down