vbatts/llama.cpp
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

omit tokenization when training is disabled, only save llama lora adapter

Browse source 
training can be disabled by passing '-n 0' to finetune
This commit is contained in:
xaedes 2023-08-29 19:21:09 +02:00
parent ebff3a14c3
commit 5813ac832f
No known key found for this signature in database
GPG key ID: 30030EDD817EA2B1
1 changed files with 17 additions and 13 deletions
Download patch file Download diff file Expand all files Collapse all files

30
examples/finetune/finetune.cpp
View file

@ -2457,12 +2457,14 @@ int main(int argc, char ** argv) {
struct llama_model * lmodel = llama_load_model_from_file(params.fn_model_base, llama_params); struct llama_model * lmodel = llama_load_model_from_file(params.fn_model_base, llama_params);
struct llama_context * lctx = llama_new_context_with_model(lmodel, llama_params); struct llama_context * lctx = llama_new_context_with_model(lmodel, llama_params);
printf("%s: tokenize training data\n", __func__);
std::vector<llama_token> train_tokens; std::vector<llama_token> train_tokens;
if (tokenize_file(lctx, params.fn_train_data, train_tokens) < 0) { if (params.n_examples > 0) {
fprintf(stderr, "%s: failed to tokenize file '%s'\n", __func__, params.fn_train_data); printf("%s: tokenize training data\n", __func__);
if (tokenize_file(lctx, params.fn_train_data, train_tokens) < 0) {
fprintf(stderr, "%s: failed to tokenize file '%s'\n", __func__, params.fn_train_data);
}
printf("%s: number of training tokens: %d\n", __func__, (int) train_tokens.size());
} }
printf("%s: number of training tokens: %d\n", __func__, (int) train_tokens.size());
struct my_llama_model model; struct my_llama_model model;
init_model(lmodel, &model, params.n_ctx); init_model(lmodel, &model, params.n_ctx);
@ -2579,17 +2581,19 @@ int main(int argc, char ** argv) {
alloc = ggml_allocr_new(compute_buf_0, size_buf_0, tensor_alignment); alloc = ggml_allocr_new(compute_buf_0, size_buf_0, tensor_alignment);
} }
GGML_ASSERT(n_tokens < (int) train_tokens.size());
std::vector<int> train_samples; std::vector<int> train_samples;
train_samples.push_back(0); if (params.n_examples > 0) {
for (int i = 1; i < (int) train_tokens.size() - n_tokens; ++i) { GGML_ASSERT(n_tokens < (int) train_tokens.size());
if (!params.samples_start_after_nl || (train_tokens[i-1] == llama_token_nl(lctx))) { train_samples.push_back(0);
train_samples.push_back(i); for (int i = 1; i < (int) train_tokens.size() - n_tokens; ++i) {
if (!params.samples_start_after_nl || (train_tokens[i-1] == llama_token_nl(lctx))) {
train_samples.push_back(i);
}
}
shuffle_ints(train_samples.data(), train_samples.data() + train_samples.size());
for (int i = 0; i < (int) train_samples.size(); ++i) {
GGML_ASSERT(train_samples[i]+n_tokens-1 < (int) train_tokens.size());
} }
}
shuffle_ints(train_samples.data(), train_samples.data() + train_samples.size());
for (int i = 0; i < (int) train_samples.size(); ++i) {
GGML_ASSERT(train_samples[i]+n_tokens-1 < (int) train_tokens.size());
} }
printf("%s: begin training\n", __func__); printf("%s: begin training\n", __func__);