vbatts/llama.cpp
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

add input tensors as checkpoints

Browse source 
so that recursive tensor cloning of gradient checkpointing terminates on input tensors
This commit is contained in:
xaedes 2023-08-14 17:58:49 +02:00
parent b2f1310196
commit 5884b43a62
No known key found for this signature in database
GPG key ID: 30030EDD817EA2B1
1 changed files with 4 additions and 1 deletions
Download patch file Download diff file Expand all files Collapse all files

5
examples/train-text-from-scratch/train-text-from-scratch.cpp
View file

@ -1579,7 +1579,10 @@ struct ggml_tensor * llama_build_train_graphs(
struct ggml_tensor * cur = t01; struct ggml_tensor * cur = t01;
std::vector<struct ggml_tensor *> checkpoints; std::vector<struct ggml_tensor *> checkpoints;
checkpoints.push_back(cur); checkpoints.push_back(tokens_input);
checkpoints.push_back(targets);
checkpoints.push_back(t00);
checkpoints.push_back(t01);
struct ggml_tensor * kv_scale; struct ggml_tensor * kv_scale;
if (flash_attn) { if (flash_attn) {