vbatts/llama.cpp
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

vastly improve training results

Browse source 
instead of logit targets 0 and 1 use -1 and +1.
This commit is contained in:
xaedes 2023-05-07 19:46:05 +02:00
parent 93201abdb7
commit 49d6daa11e
No known key found for this signature in database
GPG key ID: 30030EDD817EA2B1
1 changed files with 2 additions and 1 deletions
Download patch file Download diff file Expand all files Collapse all files

3
examples/baby-llama/baby-llama.cpp
View file

@ -1058,7 +1058,8 @@ void get_example_targets(int example_id, struct ggml_tensor * tokens_input, stru
int n_tokens = tokens_input->ne[0];
int n_vocab = targets->ne[0];
float randomness = 0.0f;
ggml_set_zero(targets);
// ggml_set_zero(targets);
ggml_set_f32(targets, -1.0f);
ggml_set_i32_1d(tokens_input, 0, 0);
for (int i=1; i<n_tokens+1; ++i) {
float x = example_id + i * 3.14159f * 2.0f * 1.0f / n_tokens;