vbatts/llama.cpp
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Transpose after setting data

Browse source
This commit is contained in:
Lorenzo Toniazzi 2024-07-07 10:32:53 +01:00
parent 931134b536
commit 41e8c733f6
1 changed files with 8 additions and 6 deletions
Download patch file Download diff file Expand all files Collapse all files

14
llama.cpp
View file

@ -328,12 +328,6 @@ static struct lora_data * load_lora(struct lora_info * info) {
file.seek((0-file.tell()) & 31, SEEK_CUR); file.seek((0-file.tell()) & 31, SEEK_CUR);
size_t offset = file.tell(); size_t offset = file.tell();
struct ggml_tensor * tensor = ggml_new_tensor(result->ctx, (enum ggml_type) type, n_dims, ne); struct ggml_tensor * tensor = ggml_new_tensor(result->ctx, (enum ggml_type) type, n_dims, ne);
// Transpose lora matrix A
if (std::string(name_buf.data()).find("loraA") != std::string::npos) {
tensor = ggml_cont(result->ctx,
ggml_transpose(result->ctx, tensor)
);
}
ggml_set_name(tensor, name_buf.data()); ggml_set_name(tensor, name_buf.data());
size_t nbytes = ggml_nbytes(tensor); size_t nbytes = ggml_nbytes(tensor);
size_t nbytes_pad = ggml_nbytes_pad(tensor); size_t nbytes_pad = ggml_nbytes_pad(tensor);
@ -360,6 +354,14 @@ static struct lora_data * load_lora(struct lora_info * info) {
read_buf.resize(nbytes); read_buf.resize(nbytes);
file.read_raw(read_buf.data(), nbytes); file.read_raw(read_buf.data(), nbytes);
ggml_backend_tensor_set(tensor, read_buf.data(), 0, nbytes); ggml_backend_tensor_set(tensor, read_buf.data(), 0, nbytes);
// Transpose lora matrix A
std::string original_name(tensor->name);
if (std::string(tensor->name).find(".loraA") != std::string::npos) {
tensor = ggml_cont(result->ctx,
ggml_transpose(result->ctx, tensor)
);
ggml_set_name(tensor, original_name.c_str());
}
} }
return result; return result;
} }