vbatts/llama.cpp
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

correct comment

Browse source
This commit is contained in:
Xuan Son Nguyen 2025-01-07 22:55:04 +01:00
parent b37af1424a
commit 0615cdd7a4
2 changed files with 3 additions and 3 deletions
Download patch file Download diff file Expand all files Collapse all files

4
convert_lora_to_gguf.py
View file

@ -433,8 +433,8 @@ if __name__ == '__main__':
assert isinstance(dest_data, LoraTorchTensor) assert isinstance(dest_data, LoraTorchTensor)
lora_a, lora_b = dest_data.get_lora_A_B() lora_a, lora_b = dest_data.get_lora_A_B()
# token_embd A and B are already transposed by mergekit-extract-lora # note: mergekit-extract-lora flip and transpose A and B
# we transpose A back again because it is used by llm_build_inp_embd() # here we only need to transpose token_embd.lora_a, see llm_build_inp_embd()
if "token_embd.weight" in dest_name: if "token_embd.weight" in dest_name:
lora_a = lora_a.T lora_a = lora_a.T

2
src/llama-adapter.cpp
View file

@ -270,7 +270,7 @@ static void llama_lora_adapter_init_impl(struct llama_model & model, const char
struct ggml_context * dev_ctx = ctx_for_buft(ggml_backend_buffer_get_type(model_tensor->buffer)); struct ggml_context * dev_ctx = ctx_for_buft(ggml_backend_buffer_get_type(model_tensor->buffer));
// validate tensor shape // validate tensor shape
if (is_token_embd) { if (is_token_embd) {
// expect B to be transposed, see llm_build_inp_embd() // expect B to be non-transposed, A and B are flipped; see llm_build_inp_embd()
if (model_tensor->ne[0] != w.b->ne[1] || model_tensor->ne[1] != w.a->ne[1]) { if (model_tensor->ne[0] != w.b->ne[1] || model_tensor->ne[1] != w.a->ne[1]) {
throw std::runtime_error("tensor '" + name + "' has incorrect shape"); throw std::runtime_error("tensor '" + name + "' has incorrect shape");
} }