vbatts/llama.cpp
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

finetune: add training data file to log message

Browse source 
This commit adds the name of the training data file to the log message
printed when the training data is tokenized.

The motivation for this change is that it can be useful to show which
file is being tokenized when running the finetune example.

Signed-off-by: Daniel Bevenius <daniel.bevenius@gmail.com>
This commit is contained in:
Daniel Bevenius 2024-01-16 15:34:48 +01:00
parent 3e5ca7931c
commit 0df8b949ce
Failed to extract signature
1 changed files with 1 additions and 1 deletions
Download patch file Download diff file Expand all files Collapse all files

2
examples/finetune/finetune.cpp
View file

@ -1800,7 +1800,7 @@ int main(int argc, char ** argv) {
std::vector<llama_token> train_tokens; std::vector<llama_token> train_tokens;
std::vector<size_t> train_samples_begin; std::vector<size_t> train_samples_begin;
std::vector<size_t> train_samples_size; std::vector<size_t> train_samples_size;
printf("%s: tokenize training data\n", __func__); printf("%s: tokenize training data from %s\n", __func__, params.common.fn_train_data);
tokenize_file(lctx, tokenize_file(lctx,
params.common.fn_train_data, params.common.fn_train_data,
params.common.sample_start, params.common.sample_start,