vbatts/llama.cpp
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

fix random weight initialization scale

Browse source
This commit is contained in:
xaedes 2023-05-21 12:18:47 +02:00
parent 96514971dd
commit 57c2f4f909
No known key found for this signature in database
GPG key ID: 30030EDD817EA2B1
1 changed files with 3 additions and 3 deletions
Download patch file Download diff file Expand all files Collapse all files

6
examples/baby-llama/baby-llama-text.cpp
View file

@ -68,7 +68,7 @@ struct ggml_tensor * randomize_tensor_normal(struct ggml_tensor * tensor, struct
} }
break; break;
case 2: case 2:
scale /= sqrtf(tensor->ne[0]*tensor->ne[1]); scale /= sqrtf(tensor->ne[0]+tensor->ne[1]);
for (int i1 = 0; i1 < tensor->ne[1]; i1++) { for (int i1 = 0; i1 < tensor->ne[1]; i1++) {
for (int i0 = 0; i0 < tensor->ne[0]; i0++) { for (int i0 = 0; i0 < tensor->ne[0]; i0++) {
float * dst = (float *) ((char *) tensor->data + i0*tensor->nb[0] + i1*tensor->nb[1]); float * dst = (float *) ((char *) tensor->data + i0*tensor->nb[0] + i1*tensor->nb[1]);
@ -77,7 +77,7 @@ struct ggml_tensor * randomize_tensor_normal(struct ggml_tensor * tensor, struct
} }
break; break;
case 3: case 3:
scale /= sqrtf(tensor->ne[0]*tensor->ne[1]); scale /= sqrtf(tensor->ne[0]+tensor->ne[1]);
for (int i2 = 0; i2 < tensor->ne[2]; i2++) { for (int i2 = 0; i2 < tensor->ne[2]; i2++) {
for (int i1 = 0; i1 < tensor->ne[1]; i1++) { for (int i1 = 0; i1 < tensor->ne[1]; i1++) {
for (int i0 = 0; i0 < tensor->ne[0]; i0++) { for (int i0 = 0; i0 < tensor->ne[0]; i0++) {
@ -88,7 +88,7 @@ struct ggml_tensor * randomize_tensor_normal(struct ggml_tensor * tensor, struct
} }
break; break;
case 4: case 4:
scale /= sqrtf(tensor->ne[0]*tensor->ne[1]); scale /= sqrtf(tensor->ne[0]+tensor->ne[1]);
for (int i3 = 0; i3 < tensor->ne[3]; i3++) { for (int i3 = 0; i3 < tensor->ne[3]; i3++) {
for (int i2 = 0; i2 < tensor->ne[2]; i2++) { for (int i2 = 0; i2 < tensor->ne[2]; i2++) {
for (int i1 = 0; i1 < tensor->ne[1]; i1++) { for (int i1 = 0; i1 < tensor->ne[1]; i1++) {