vbatts/llama.cpp
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

YaRN : correction to GPT-NeoX implementation

Browse source 
ggml-ci
This commit is contained in:
Jared Van Bortel 2023-11-15 17:07:57 -05:00
parent a6fc554e26
commit f824902623
3 changed files with 7 additions and 12 deletions
Download patch file Download diff file Expand all files Collapse all files

6
ggml.c
View file

@ -11084,8 +11084,7 @@ static void ggml_compute_forward_rope_f32(
theta_base *= freq_scale;
for (int64_t ib = 0; ib < ne0/n_dims; ++ib) {
for (int64_t ic = 0; ic < n_dims; ic += 2) {
// simplified from `(ib * n_dims + ic) * inv_ndims`
float cur_rot = inv_ndims * ic - ib;
int64_t cur_rot = ib * n_dims + ic;
float cos_theta, sin_theta;
rope_yarn(
@ -11237,8 +11236,7 @@ static void ggml_compute_forward_rope_f16(
theta_base *= freq_scale;
for (int64_t ib = 0; ib < ne0/n_dims; ++ib) {
for (int64_t ic = 0; ic < n_dims; ic += 2) {
// simplified from `(ib * n_dims + ic) * inv_ndims`
float cur_rot = inv_ndims * ic - ib;
int64_t cur_rot = ib * n_dims + ic;
float cos_theta, sin_theta;
rope_yarn(