vbatts/llama.cpp
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

addlocate gpt_params on heap instead to avoid rare segfault

Browse source
This commit is contained in:
Concedo 2023-12-28 11:48:21 +08:00
parent 69ab1bf2f8
commit 2d5d82e915
2 changed files with 132 additions and 111 deletions
Download patch file Download diff file Expand all files Collapse all files

223
gpttype_adapter.cpp
View file

@ -73,7 +73,7 @@ static llama_v2_context * llama_ctx_v2;
static llama_v3_context * llama_ctx_v3; static llama_v3_context * llama_ctx_v3;
static llama_context * llama_ctx_v4; static llama_context * llama_ctx_v4;
static gpt_params params; static gpt_params * kcpp_params;
static int max_context_limit_at_load = 0; static int max_context_limit_at_load = 0;
static int n_past = 0; static int n_past = 0;
static int n_threads = 4; static int n_threads = 4;
@ -677,14 +677,15 @@ void PurgeMissingTokens(llama_context * ctx, std::vector<int> &current_context_t
ModelLoadResult gpttype_load_model(const load_model_inputs inputs, FileFormat in_file_format, FileFormatExtraMeta file_format_meta) ModelLoadResult gpttype_load_model(const load_model_inputs inputs, FileFormat in_file_format, FileFormatExtraMeta file_format_meta)
{ {
ggml_time_init(); ggml_time_init();
kcpp_params = new gpt_params(); //allocate on heap to avoid linux segfault. yes this leaks memory.
file_format = in_file_format; file_format = in_file_format;
n_threads = params.n_threads = inputs.threads; n_threads = kcpp_params->n_threads = inputs.threads;
n_blasthreads = params.n_threads_batch = inputs.blasthreads; n_blasthreads = kcpp_params->n_threads_batch = inputs.blasthreads;
bool isGguf = (file_format == FileFormat::GGUF_LLAMA || file_format==FileFormat::GGUF_FALCON); bool isGguf = (file_format == FileFormat::GGUF_LLAMA || file_format==FileFormat::GGUF_FALCON);
n_batch = params.n_batch = (isGguf?normalbatchsize:smallbatchsize); n_batch = kcpp_params->n_batch = (isGguf?normalbatchsize:smallbatchsize);
modelname = params.model = inputs.model_filename; modelname = kcpp_params->model = inputs.model_filename;
useSmartContext = inputs.use_smartcontext; useSmartContext = inputs.use_smartcontext;
useContextShift = inputs.use_contextshift; useContextShift = inputs.use_contextshift;
debugmode = inputs.debugmode; debugmode = inputs.debugmode;
@ -703,13 +704,13 @@ ModelLoadResult gpttype_load_model(const load_model_inputs inputs, FileFormat in
clamped_max_context_length = 16384; clamped_max_context_length = 16384;
} }
params.n_ctx = clamped_max_context_length; kcpp_params->n_ctx = clamped_max_context_length;
max_context_limit_at_load = clamped_max_context_length; max_context_limit_at_load = clamped_max_context_length;
neox_ctx_v2.hparams.n_ctx = neox_ctx_v3.hparams.n_ctx neox_ctx_v2.hparams.n_ctx = neox_ctx_v3.hparams.n_ctx
= gptj_ctx_v1.hparams.n_ctx = gptj_ctx_v2.hparams.n_ctx = gptj_ctx_v3.hparams.n_ctx = gptj_ctx_v1.hparams.n_ctx = gptj_ctx_v2.hparams.n_ctx = gptj_ctx_v3.hparams.n_ctx
= gpt2_ctx_v1.hparams.n_ctx = gpt2_ctx_v2.hparams.n_ctx = gpt2_ctx_v3.hparams.n_ctx = gpt2_ctx_v1.hparams.n_ctx = gpt2_ctx_v2.hparams.n_ctx = gpt2_ctx_v3.hparams.n_ctx
= mpt_ctx_v3.hparams.n_ctx = params.n_ctx; = mpt_ctx_v3.hparams.n_ctx = kcpp_params->n_ctx;
//determine rope scaling params //determine rope scaling params
float rope_freq_scale = 1.0f; float rope_freq_scale = 1.0f;
@ -725,14 +726,14 @@ ModelLoadResult gpttype_load_model(const load_model_inputs inputs, FileFormat in
else else
{ {
rope_freq_scale = 1.0f; rope_freq_scale = 1.0f;
if (params.n_ctx <= 2048) //normie mode if (kcpp_params->n_ctx <= 2048) //normie mode
{ {
rope_freq_base = 10000.0f; rope_freq_base = 10000.0f;
} }
else else
{ {
//approximate NTK aware ctx //approximate NTK aware ctx
auto effectivenctx = params.n_ctx; auto effectivenctx = kcpp_params->n_ctx;
if((file_format == FileFormat::GGUF_LLAMA || file_format==FileFormat::GGUF_FALCON) && file_format_meta.n_ctx_train > 2048) if((file_format == FileFormat::GGUF_LLAMA || file_format==FileFormat::GGUF_FALCON) && file_format_meta.n_ctx_train > 2048)
{ {
float factor = file_format_meta.n_ctx_train/2048; float factor = file_format_meta.n_ctx_train/2048;
@ -822,7 +823,7 @@ ModelLoadResult gpttype_load_model(const load_model_inputs inputs, FileFormat in
//determine mem per token //determine mem per token
const std::vector<int> tmp = {1, 2, 3, 4}; const std::vector<int> tmp = {1, 2, 3, 4};
llama_v2_eval(llama_ctx_v2, tmp.data(), tmp.size(), 0, params.n_threads); llama_v2_eval(llama_ctx_v2, tmp.data(), tmp.size(), 0, kcpp_params->n_threads);
return ModelLoadResult::SUCCESS; return ModelLoadResult::SUCCESS;
} }
else if(file_format == FileFormat::GGJT_3) else if(file_format == FileFormat::GGJT_3)
@ -889,7 +890,7 @@ ModelLoadResult gpttype_load_model(const load_model_inputs inputs, FileFormat in
//determine mem per token //determine mem per token
const std::vector<int> tmp = {1, 2, 3, 4}; const std::vector<int> tmp = {1, 2, 3, 4};
auto er = llama_v3_eval(llama_ctx_v3, tmp.data(), tmp.size(), 0, params.n_threads); auto er = llama_v3_eval(llama_ctx_v3, tmp.data(), tmp.size(), 0, kcpp_params->n_threads);
if(er!=0) if(er!=0)
{ {
printf("\nLLAMA EVAL returned nonzero!\n"); printf("\nLLAMA EVAL returned nonzero!\n");
@ -1119,7 +1120,7 @@ ModelLoadResult gpttype_load_model(const load_model_inputs inputs, FileFormat in
rwkv_ctx_v3->logits_out = (float *)malloc(logitbufsiz); rwkv_ctx_v3->logits_out = (float *)malloc(logitbufsiz);
rwkv_ctx_v3->state_in = nullptr; rwkv_ctx_v3->state_in = nullptr;
bool testeval = rwkv_eval(rwkv_ctx_v3, params.n_threads, 0, rwkv_ctx_v3->state_in, rwkv_ctx_v3->state_out, rwkv_ctx_v3->logits_out); bool testeval = rwkv_eval(rwkv_ctx_v3, kcpp_params->n_threads, 0, rwkv_ctx_v3->state_in, rwkv_ctx_v3->state_out, rwkv_ctx_v3->logits_out);
if (!testeval) if (!testeval)
{ {
printf("\nError: RWKV Init Eval Failed!\n"); printf("\nError: RWKV Init Eval Failed!\n");
@ -1136,10 +1137,10 @@ ModelLoadResult gpttype_load_model(const load_model_inputs inputs, FileFormat in
} }
else if (file_format == FileFormat::GPT2_1) else if (file_format == FileFormat::GPT2_1)
{ {
ModelLoadResult res = legacy_gpt2_model_load(params.model, gpt2_ctx_v1, vocab, file_format); ModelLoadResult res = legacy_gpt2_model_load(kcpp_params->model, gpt2_ctx_v1, vocab, file_format);
if(res==ModelLoadResult::FAIL) if(res==ModelLoadResult::FAIL)
{ {
fprintf(stderr, "%s: failed to load model from '%s'\n", __func__, params.model.c_str()); fprintf(stderr, "%s: failed to load model from '%s'\n", __func__, kcpp_params->model.c_str());
return res; return res;
} }
else if(res==ModelLoadResult::RETRY_LOAD) else if(res==ModelLoadResult::RETRY_LOAD)
@ -1151,17 +1152,17 @@ ModelLoadResult gpttype_load_model(const load_model_inputs inputs, FileFormat in
n_vocab = gpt2_ctx_v1.hparams.n_vocab; n_vocab = gpt2_ctx_v1.hparams.n_vocab;
// determine the required inference memory per token: // determine the required inference memory per token:
legacy_gpt2_eval(gpt2_ctx_v1, params.n_threads, 0, { 0, 1, 2, 3 }, logits, mem_per_token, file_format); legacy_gpt2_eval(gpt2_ctx_v1, kcpp_params->n_threads, 0, { 0, 1, 2, 3 }, logits, mem_per_token, file_format);
return ModelLoadResult::SUCCESS; return ModelLoadResult::SUCCESS;
} }
else if (file_format == FileFormat::GPT2_2 || file_format==FileFormat::GPT2_3 || file_format==FileFormat::GPT2_4) else if (file_format == FileFormat::GPT2_2 || file_format==FileFormat::GPT2_3 || file_format==FileFormat::GPT2_4)
{ {
if(file_format==FileFormat::GPT2_4) if(file_format==FileFormat::GPT2_4)
{ {
ModelLoadResult res = gpt2_model_load(params.model, gpt2_ctx_v3, vocab, file_format, inputs.gpulayers); ModelLoadResult res = gpt2_model_load(kcpp_params->model, gpt2_ctx_v3, vocab, file_format, inputs.gpulayers);
if(res==ModelLoadResult::FAIL) if(res==ModelLoadResult::FAIL)
{ {
fprintf(stderr, "%s: failed to load model from '%s'\n", __func__, params.model.c_str()); fprintf(stderr, "%s: failed to load model from '%s'\n", __func__, kcpp_params->model.c_str());
return res; return res;
} }
else if(res==ModelLoadResult::RETRY_LOAD) else if(res==ModelLoadResult::RETRY_LOAD)
@ -1173,7 +1174,7 @@ ModelLoadResult gpttype_load_model(const load_model_inputs inputs, FileFormat in
n_vocab = gpt2_ctx_v3.hparams.n_vocab; n_vocab = gpt2_ctx_v3.hparams.n_vocab;
// determine the required inference memory per token: // determine the required inference memory per token:
gpt2_eval(gpt2_ctx_v3, params.n_threads, 0, { 0, 1, 2, 3 }, logits, mem_per_token, use_scratch); gpt2_eval(gpt2_ctx_v3, kcpp_params->n_threads, 0, { 0, 1, 2, 3 }, logits, mem_per_token, use_scratch);
return ModelLoadResult::SUCCESS; return ModelLoadResult::SUCCESS;
} }
else else
@ -1181,10 +1182,10 @@ ModelLoadResult gpttype_load_model(const load_model_inputs inputs, FileFormat in
//newer format has bit unshuffling //newer format has bit unshuffling
SetQuantsUnshuffled(file_format == FileFormat::GPT2_3); SetQuantsUnshuffled(file_format == FileFormat::GPT2_3);
ModelLoadResult res = gpt2_v2_model_load(params.model, gpt2_ctx_v2, vocab, file_format, inputs.gpulayers); ModelLoadResult res = gpt2_v2_model_load(kcpp_params->model, gpt2_ctx_v2, vocab, file_format, inputs.gpulayers);
if(res==ModelLoadResult::FAIL) if(res==ModelLoadResult::FAIL)
{ {
fprintf(stderr, "%s: failed to load model from '%s'\n", __func__, params.model.c_str()); fprintf(stderr, "%s: failed to load model from '%s'\n", __func__, kcpp_params->model.c_str());
return res; return res;
} }
else if(res==ModelLoadResult::RETRY_LOAD) else if(res==ModelLoadResult::RETRY_LOAD)
@ -1196,16 +1197,16 @@ ModelLoadResult gpttype_load_model(const load_model_inputs inputs, FileFormat in
n_vocab = gpt2_ctx_v2.hparams.n_vocab; n_vocab = gpt2_ctx_v2.hparams.n_vocab;
// determine the required inference memory per token: // determine the required inference memory per token:
gpt2_v2_eval(gpt2_ctx_v2, params.n_threads, 0, { 0, 1, 2, 3 }, logits, mem_per_token, file_format); gpt2_v2_eval(gpt2_ctx_v2, kcpp_params->n_threads, 0, { 0, 1, 2, 3 }, logits, mem_per_token, file_format);
return ModelLoadResult::SUCCESS; return ModelLoadResult::SUCCESS;
} }
} }
else if (file_format == FileFormat::GPTJ_1 || file_format == FileFormat::GPTJ_2) else if (file_format == FileFormat::GPTJ_1 || file_format == FileFormat::GPTJ_2)
{ {
ModelLoadResult res = legacy_gptj_model_load(params.model, gptj_ctx_v1, vocab, file_format); ModelLoadResult res = legacy_gptj_model_load(kcpp_params->model, gptj_ctx_v1, vocab, file_format);
if(res==ModelLoadResult::FAIL) if(res==ModelLoadResult::FAIL)
{ {
fprintf(stderr, "%s: failed to load model from '%s'\n", __func__, params.model.c_str()); fprintf(stderr, "%s: failed to load model from '%s'\n", __func__, kcpp_params->model.c_str());
return res; return res;
} }
else if(res==ModelLoadResult::RETRY_LOAD) else if(res==ModelLoadResult::RETRY_LOAD)
@ -1217,7 +1218,7 @@ ModelLoadResult gpttype_load_model(const load_model_inputs inputs, FileFormat in
n_vocab = gptj_ctx_v1.hparams.n_vocab; n_vocab = gptj_ctx_v1.hparams.n_vocab;
// determine the required inference memory per token: // determine the required inference memory per token:
legacy_gptj_eval(gptj_ctx_v1, params.n_threads, 0, { 0, 1, 2, 3 }, logits, mem_per_token, file_format); legacy_gptj_eval(gptj_ctx_v1, kcpp_params->n_threads, 0, { 0, 1, 2, 3 }, logits, mem_per_token, file_format);
//if the logits are NAN or duplicated, it means the model is incompatible //if the logits are NAN or duplicated, it means the model is incompatible
if(logits.size()>0 && IsNanCheck(logits[0])) if(logits.size()>0 && IsNanCheck(logits[0]))
@ -1233,10 +1234,10 @@ ModelLoadResult gpttype_load_model(const load_model_inputs inputs, FileFormat in
{ {
if(file_format == FileFormat::GPTJ_5) if(file_format == FileFormat::GPTJ_5)
{ {
ModelLoadResult loadresult = gptj_model_load(params.model, gptj_ctx_v3, vocab, inputs.gpulayers); ModelLoadResult loadresult = gptj_model_load(kcpp_params->model, gptj_ctx_v3, vocab, inputs.gpulayers);
if (loadresult == ModelLoadResult::FAIL) if (loadresult == ModelLoadResult::FAIL)
{ {
fprintf(stderr, "%s: failed to load model from '%s'\n", __func__, params.model.c_str()); fprintf(stderr, "%s: failed to load model from '%s'\n", __func__, kcpp_params->model.c_str());
return loadresult; return loadresult;
} }
else if (loadresult == ModelLoadResult::RETRY_LOAD) else if (loadresult == ModelLoadResult::RETRY_LOAD)
@ -1248,14 +1249,14 @@ ModelLoadResult gpttype_load_model(const load_model_inputs inputs, FileFormat in
n_vocab = gptj_ctx_v3.hparams.n_vocab; n_vocab = gptj_ctx_v3.hparams.n_vocab;
// determine the required inference memory per token: // determine the required inference memory per token:
gptj_eval(gptj_ctx_v3, params.n_threads, 0, { 0, 1, 2, 3 }, logits, mem_per_token, use_scratch); gptj_eval(gptj_ctx_v3, kcpp_params->n_threads, 0, { 0, 1, 2, 3 }, logits, mem_per_token, use_scratch);
//if the logits are NAN or duplicated, it means the model is incompatible //if the logits are NAN or duplicated, it means the model is incompatible
std::vector<float> oldlogits(logits); std::vector<float> oldlogits(logits);
//this is another hack because they change the library - we run the eval through the model //this is another hack because they change the library - we run the eval through the model
//twice and compare logits. if they give the same logits for different inputs, model is broken //twice and compare logits. if they give the same logits for different inputs, model is broken
gptj_eval(gptj_ctx_v3, params.n_threads, 0, {4, 5, 6, 7}, logits, mem_per_token, use_scratch); gptj_eval(gptj_ctx_v3, kcpp_params->n_threads, 0, {4, 5, 6, 7}, logits, mem_per_token, use_scratch);
if(logits.size()>0 && (IsNanCheck(logits[0]) || LogitsDuplicated(oldlogits,logits))) if(logits.size()>0 && (IsNanCheck(logits[0]) || LogitsDuplicated(oldlogits,logits)))
{ {
@ -1271,10 +1272,10 @@ ModelLoadResult gpttype_load_model(const load_model_inputs inputs, FileFormat in
//newer format has bit unshuffling //newer format has bit unshuffling
SetQuantsUnshuffled(file_format == FileFormat::GPTJ_4); SetQuantsUnshuffled(file_format == FileFormat::GPTJ_4);
ModelLoadResult loadresult = gptj_v2_model_load(params.model, gptj_ctx_v2, vocab, inputs.gpulayers); ModelLoadResult loadresult = gptj_v2_model_load(kcpp_params->model, gptj_ctx_v2, vocab, inputs.gpulayers);
if (loadresult == ModelLoadResult::FAIL) if (loadresult == ModelLoadResult::FAIL)
{ {
fprintf(stderr, "%s: failed to load model from '%s'\n", __func__, params.model.c_str()); fprintf(stderr, "%s: failed to load model from '%s'\n", __func__, kcpp_params->model.c_str());
return loadresult; return loadresult;
} }
else if (loadresult == ModelLoadResult::RETRY_LOAD) else if (loadresult == ModelLoadResult::RETRY_LOAD)
@ -1286,14 +1287,14 @@ ModelLoadResult gpttype_load_model(const load_model_inputs inputs, FileFormat in
n_vocab = gptj_ctx_v2.hparams.n_vocab; n_vocab = gptj_ctx_v2.hparams.n_vocab;
// determine the required inference memory per token: // determine the required inference memory per token:
gptj_v2_eval(gptj_ctx_v2, params.n_threads, 0, { 0, 1, 2, 3 }, logits, mem_per_token); gptj_v2_eval(gptj_ctx_v2, kcpp_params->n_threads, 0, { 0, 1, 2, 3 }, logits, mem_per_token);
//if the logits are NAN or duplicated, it means the model is incompatible //if the logits are NAN or duplicated, it means the model is incompatible
std::vector<float> oldlogits(logits); std::vector<float> oldlogits(logits);
//this is another hack because they change the library - we run the eval through the model //this is another hack because they change the library - we run the eval through the model
//twice and compare logits. if they give the same logits for different inputs, model is broken //twice and compare logits. if they give the same logits for different inputs, model is broken
gptj_v2_eval(gptj_ctx_v2, params.n_threads, 0, {4, 5, 6, 7}, logits, mem_per_token); gptj_v2_eval(gptj_ctx_v2, kcpp_params->n_threads, 0, {4, 5, 6, 7}, logits, mem_per_token);
if(logits.size()>0 && (IsNanCheck(logits[0]) || LogitsDuplicated(oldlogits,logits))) if(logits.size()>0 && (IsNanCheck(logits[0]) || LogitsDuplicated(oldlogits,logits)))
{ {
@ -1309,10 +1310,10 @@ ModelLoadResult gpttype_load_model(const load_model_inputs inputs, FileFormat in
{ {
if(file_format==FileFormat::NEOX_6|| file_format==FileFormat::NEOX_7) if(file_format==FileFormat::NEOX_6|| file_format==FileFormat::NEOX_7)
{ {
ModelLoadResult res = gpt_neox_model_load(params.model, neox_ctx_v3, vocab, file_format, inputs.gpulayers); ModelLoadResult res = gpt_neox_model_load(kcpp_params->model, neox_ctx_v3, vocab, file_format, inputs.gpulayers);
if(res==ModelLoadResult::FAIL) if(res==ModelLoadResult::FAIL)
{ {
fprintf(stderr, "%s: failed to load model from '%s'\n", __func__, params.model.c_str()); fprintf(stderr, "%s: failed to load model from '%s'\n", __func__, kcpp_params->model.c_str());
return res; return res;
} }
else if(res==ModelLoadResult::RETRY_LOAD) else if(res==ModelLoadResult::RETRY_LOAD)
@ -1324,7 +1325,7 @@ ModelLoadResult gpttype_load_model(const load_model_inputs inputs, FileFormat in
n_vocab = neox_ctx_v3.hparams.n_vocab; n_vocab = neox_ctx_v3.hparams.n_vocab;
// determine the required inference memory per token: // determine the required inference memory per token:
gpt_neox_eval(neox_ctx_v3, params.n_threads, 0, { 0, 1, 2, 3 }, logits, mem_per_token, use_scratch); gpt_neox_eval(neox_ctx_v3, kcpp_params->n_threads, 0, { 0, 1, 2, 3 }, logits, mem_per_token, use_scratch);
return ModelLoadResult::SUCCESS; return ModelLoadResult::SUCCESS;
} }
@ -1333,10 +1334,10 @@ ModelLoadResult gpttype_load_model(const load_model_inputs inputs, FileFormat in
//newer format has bit unshuffling //newer format has bit unshuffling
SetQuantsUnshuffled(file_format==FileFormat::NEOX_4 || file_format==FileFormat::NEOX_5); SetQuantsUnshuffled(file_format==FileFormat::NEOX_4 || file_format==FileFormat::NEOX_5);
ModelLoadResult res = gpt_neox_v2_model_load(params.model, neox_ctx_v2, vocab, file_format); ModelLoadResult res = gpt_neox_v2_model_load(kcpp_params->model, neox_ctx_v2, vocab, file_format);
if(res==ModelLoadResult::FAIL) if(res==ModelLoadResult::FAIL)
{ {
fprintf(stderr, "%s: failed to load model from '%s'\n", __func__, params.model.c_str()); fprintf(stderr, "%s: failed to load model from '%s'\n", __func__, kcpp_params->model.c_str());
return res; return res;
} }
else if(res==ModelLoadResult::RETRY_LOAD) else if(res==ModelLoadResult::RETRY_LOAD)
@ -1348,7 +1349,7 @@ ModelLoadResult gpttype_load_model(const load_model_inputs inputs, FileFormat in
n_vocab = neox_ctx_v2.hparams.n_vocab; n_vocab = neox_ctx_v2.hparams.n_vocab;
// determine the required inference memory per token: // determine the required inference memory per token:
gpt_neox_v2_eval(neox_ctx_v2, params.n_threads, 0, { 0, 1, 2, 3 }, logits, mem_per_token); gpt_neox_v2_eval(neox_ctx_v2, kcpp_params->n_threads, 0, { 0, 1, 2, 3 }, logits, mem_per_token);
if(logits.size()>0 && file_format==FileFormat::NEOX_2 && !IsNanCheck(logits[0])) if(logits.size()>0 && file_format==FileFormat::NEOX_2 && !IsNanCheck(logits[0]))
{ {
@ -1356,7 +1357,7 @@ ModelLoadResult gpttype_load_model(const load_model_inputs inputs, FileFormat in
std::vector<int> test_embd = ::gpt_tokenize(vocab, "1 2 3 4 5 6 7"); std::vector<int> test_embd = ::gpt_tokenize(vocab, "1 2 3 4 5 6 7");
auto orig_par_res = neox_ctx_v2.hparams.par_res; auto orig_par_res = neox_ctx_v2.hparams.par_res;
neox_ctx_v2.hparams.par_res = 0; //test with residual false neox_ctx_v2.hparams.par_res = 0; //test with residual false
gpt_neox_v2_eval(neox_ctx_v2, params.n_threads, 0, test_embd, logits, mem_per_token); gpt_neox_v2_eval(neox_ctx_v2, kcpp_params->n_threads, 0, test_embd, logits, mem_per_token);
neox_ctx_v2.hparams.par_res = orig_par_res; neox_ctx_v2.hparams.par_res = orig_par_res;
int topid = std::max_element(logits.begin(),logits.end())-logits.begin(); int topid = std::max_element(logits.begin(),logits.end())-logits.begin();
std::string predicted = vocab.id_to_token[topid].c_str(); std::string predicted = vocab.id_to_token[topid].c_str();
@ -1375,17 +1376,17 @@ ModelLoadResult gpttype_load_model(const load_model_inputs inputs, FileFormat in
} }
else if(file_format==FileFormat::MPT_1) else if(file_format==FileFormat::MPT_1)
{ {
bool res = mpt_model_load(params.model, mpt_ctx_v3, vocab, inputs.gpulayers); bool res = mpt_model_load(kcpp_params->model, mpt_ctx_v3, vocab, inputs.gpulayers);
if(res==false) if(res==false)
{ {
fprintf(stderr, "%s: failed to load model from '%s'\n", __func__, params.model.c_str()); fprintf(stderr, "%s: failed to load model from '%s'\n", __func__, kcpp_params->model.c_str());
return ModelLoadResult::FAIL; return ModelLoadResult::FAIL;
} }
n_vocab = mpt_ctx_v3.hparams.n_vocab; n_vocab = mpt_ctx_v3.hparams.n_vocab;
// determine the required inference memory per token: // determine the required inference memory per token:
mpt_eval(mpt_ctx_v3, params.n_threads, 0, { 0, 1, 2, 3 }, logits, false, mem_per_token, use_scratch); mpt_eval(mpt_ctx_v3, kcpp_params->n_threads, 0, { 0, 1, 2, 3 }, logits, false, mem_per_token, use_scratch);
return ModelLoadResult::SUCCESS; return ModelLoadResult::SUCCESS;
} }
else else
@ -1456,25 +1457,25 @@ generation_outputs gpttype_generate(const generation_inputs inputs, generation_o
} }
std::string addedmemory = inputs.memory; std::string addedmemory = inputs.memory;
params.prompt = inputs.prompt; kcpp_params->prompt = inputs.prompt;
params.seed = inputs.seed; kcpp_params->seed = inputs.seed;
params.n_predict = inputs.max_length; kcpp_params->n_predict = inputs.max_length;
params.top_k = inputs.top_k; kcpp_params->top_k = inputs.top_k;
params.top_p = inputs.top_p; kcpp_params->top_p = inputs.top_p;
params.min_p = inputs.min_p; kcpp_params->min_p = inputs.min_p;
params.typical_p = inputs.typical_p; kcpp_params->typical_p = inputs.typical_p;
params.tfs_z = inputs.tfs; kcpp_params->tfs_z = inputs.tfs;
params.temp = inputs.temperature; kcpp_params->temp = inputs.temperature;
params.repeat_last_n = inputs.rep_pen_range; kcpp_params->repeat_last_n = inputs.rep_pen_range;
params.repeat_penalty = inputs.rep_pen; kcpp_params->repeat_penalty = inputs.rep_pen;
params.presence_penalty = inputs.presence_penalty; kcpp_params->presence_penalty = inputs.presence_penalty;
params.mirostat = inputs.mirostat; kcpp_params->mirostat = inputs.mirostat;
params.mirostat_eta = inputs.mirostat_eta; kcpp_params->mirostat_eta = inputs.mirostat_eta;
params.mirostat_tau = inputs.mirostat_tau; kcpp_params->mirostat_tau = inputs.mirostat_tau;
params.n_ctx = inputs.max_context_length; kcpp_params->n_ctx = inputs.max_context_length;
params.n_batch = n_batch; kcpp_params->n_batch = n_batch;
params.n_threads = n_threads; kcpp_params->n_threads = n_threads;
params.n_threads_batch = n_blasthreads; kcpp_params->n_threads_batch = n_blasthreads;
bool stream_sse = inputs.stream_sse; bool stream_sse = inputs.stream_sse;
bool allow_regular_prints = (debugmode!=-1 && !inputs.quiet) || debugmode >= 1; bool allow_regular_prints = (debugmode!=-1 && !inputs.quiet) || debugmode >= 1;
@ -1498,37 +1499,37 @@ generation_outputs gpttype_generate(const generation_inputs inputs, generation_o
current_grammar = grammarstr; current_grammar = grammarstr;
if (params.repeat_last_n < 1) if (kcpp_params->repeat_last_n < 1)
{ {
params.repeat_last_n = 1; kcpp_params->repeat_last_n = 1;
} }
if (params.top_k < 1) if (kcpp_params->top_k < 1)
{ {
params.top_k = n_vocab; // all tokens in the vocabulary should be considered if top k is disabled kcpp_params->top_k = n_vocab; // all tokens in the vocabulary should be considered if top k is disabled
} }
if (params.seed <= 0 || params.seed==0xFFFFFFFF) if (kcpp_params->seed <= 0 || kcpp_params->seed==0xFFFFFFFF)
{ {
params.seed = time(NULL); kcpp_params->seed = time(NULL);
} }
// tokenize the prompt // tokenize the prompt
std::vector<int> embd_inp; std::vector<int> embd_inp;
std::vector<int> embd_inp_mem; //for storing added memory std::vector<int> embd_inp_mem; //for storing added memory
TokenizeString(params.prompt, embd_inp, file_format); TokenizeString(kcpp_params->prompt, embd_inp, file_format);
if(addedmemory!="") if(addedmemory!="")
{ {
TokenizeString(addedmemory, embd_inp_mem, file_format); TokenizeString(addedmemory, embd_inp_mem, file_format);
} }
//truncate to front of the prompt if its too long //truncate to front of the prompt if its too long
int32_t nctx = params.n_ctx; int32_t nctx = kcpp_params->n_ctx;
if (embd_inp.size() + params.n_predict > nctx) if (embd_inp.size() + kcpp_params->n_predict > nctx)
{ {
//get bos token //get bos token
std::vector<int> bos; std::vector<int> bos;
TokenizeString("", bos, file_format); TokenizeString("", bos, file_format);
int offset = embd_inp.size() - nctx + params.n_predict; int offset = embd_inp.size() - nctx + kcpp_params->n_predict;
embd_inp = std::vector<int>(embd_inp.begin() + offset, embd_inp.end()); embd_inp = std::vector<int>(embd_inp.begin() + offset, embd_inp.end());
//replace bos into front if exists //replace bos into front if exists
if(bos.size()>0 && embd_inp.size()>0) if(bos.size()>0 && embd_inp.size()>0)
@ -1548,9 +1549,9 @@ generation_outputs gpttype_generate(const generation_inputs inputs, generation_o
} }
//shorten memory if needed //shorten memory if needed
if (embd_inp_mem.size() + params.n_predict + 4 > nctx) if (embd_inp_mem.size() + kcpp_params->n_predict + 4 > nctx)
{ {
int offset = embd_inp_mem.size() - nctx + params.n_predict + 4; int offset = embd_inp_mem.size() - nctx + kcpp_params->n_predict + 4;
embd_inp_mem = std::vector<int>(embd_inp_mem.begin() + offset, embd_inp_mem.end()); embd_inp_mem = std::vector<int>(embd_inp_mem.begin() + offset, embd_inp_mem.end());
//replace bos into front if exists //replace bos into front if exists
if(bos.size()>0 && embd_inp_mem.size()>0) if(bos.size()>0 && embd_inp_mem.size()>0)
@ -1561,7 +1562,7 @@ generation_outputs gpttype_generate(const generation_inputs inputs, generation_o
//shorten main prompt by trimming the front if needed //shorten main prompt by trimming the front if needed
int addmemtokens = embd_inp_mem.size(); int addmemtokens = embd_inp_mem.size();
int totalsize = (addmemtokens + embd_inp.size() + params.n_predict); int totalsize = (addmemtokens + embd_inp.size() + kcpp_params->n_predict);
if(totalsize > nctx) if(totalsize > nctx)
{ {
int excess = totalsize - nctx; int excess = totalsize - nctx;
@ -1580,7 +1581,7 @@ generation_outputs gpttype_generate(const generation_inputs inputs, generation_o
//determine how much npast we have to rewind from the current state //determine how much npast we have to rewind from the current state
std::vector<gpt_vocab::id> embd; std::vector<gpt_vocab::id> embd;
int last_n_size = params.repeat_last_n; int last_n_size = kcpp_params->repeat_last_n;
last_n_tokens.resize(last_n_size); last_n_tokens.resize(last_n_size);
std::fill(last_n_tokens.begin(), last_n_tokens.end(), 0); std::fill(last_n_tokens.begin(), last_n_tokens.end(), 0);
@ -1614,8 +1615,8 @@ generation_outputs gpttype_generate(const generation_inputs inputs, generation_o
file_format==FileFormat::RWKV_2); file_format==FileFormat::RWKV_2);
bool blasmode = (approved_format && embd_inp.size() >= 32 && ggml_cpu_has_blas() && blasbatchsize!=-1); bool blasmode = (approved_format && embd_inp.size() >= 32 && ggml_cpu_has_blas() && blasbatchsize!=-1);
// bool blasmode = false; // bool blasmode = false;
int original_batch = params.n_batch; int original_batch = kcpp_params->n_batch;
int original_threads = params.n_threads; int original_threads = kcpp_params->n_threads;
if (blasmode) if (blasmode)
{ {
//for non llama, limit to 256 //for non llama, limit to 256
@ -1625,27 +1626,27 @@ generation_outputs gpttype_generate(const generation_inputs inputs, generation_o
bbs = (blasbatchsize > 256 ? 256 : blasbatchsize); bbs = (blasbatchsize > 256 ? 256 : blasbatchsize);
} }
params.n_batch = bbs; //received reports of 1024 and above crashing on some models kcpp_params->n_batch = bbs; //received reports of 1024 and above crashing on some models
if(!ggml_cpu_has_gpublas()) if(!ggml_cpu_has_gpublas())
{ {
//does not limit here for gguf anymore. this is kept for older models. //does not limit here for gguf anymore. this is kept for older models.
//new models will override threads inside decode fn. //new models will override threads inside decode fn.
params.n_threads = 1; kcpp_params->n_threads = 1;
params.n_threads_batch = 1; kcpp_params->n_threads_batch = 1;
} }
else else
{ {
params.n_threads = n_blasthreads; kcpp_params->n_threads = n_blasthreads;
params.n_threads_batch = n_blasthreads; kcpp_params->n_threads_batch = n_blasthreads;
} }
} }
current_context_tokens.resize(n_past); current_context_tokens.resize(n_past);
remaining_tokens = params.n_predict; remaining_tokens = kcpp_params->n_predict;
stopper_unused_tokens = 0; stopper_unused_tokens = 0;
int input_consumed = 0; int input_consumed = 0;
std::mt19937 rng(params.seed); std::mt19937 rng(kcpp_params->seed);
//prepare sampler order //prepare sampler order
std::vector<samplers> sampler_order; std::vector<samplers> sampler_order;
@ -1766,11 +1767,11 @@ generation_outputs gpttype_generate(const generation_inputs inputs, generation_o
if (file_format == FileFormat::GGML || file_format == FileFormat::GGHF || file_format == FileFormat::GGJT || file_format == FileFormat::GGJT_2) if (file_format == FileFormat::GGML || file_format == FileFormat::GGHF || file_format == FileFormat::GGJT || file_format == FileFormat::GGJT_2)
{ {
evalres = (llama_v2_eval(llama_ctx_v2, embd.data(), embdsize, n_past, params.n_threads)==0); evalres = (llama_v2_eval(llama_ctx_v2, embd.data(), embdsize, n_past, kcpp_params->n_threads)==0);
} }
else if(file_format == FileFormat::GGJT_3) else if(file_format == FileFormat::GGJT_3)
{ {
evalres = (llama_v3_eval(llama_ctx_v3, embd.data(), embdsize, n_past, params.n_threads)==0); evalres = (llama_v3_eval(llama_ctx_v3, embd.data(), embdsize, n_past, kcpp_params->n_threads)==0);
} }
else if(file_format == FileFormat::GGUF_LLAMA || file_format==FileFormat::GGUF_FALCON) else if(file_format == FileFormat::GGUF_LLAMA || file_format==FileFormat::GGUF_FALCON)
{ {
@ -1788,12 +1789,12 @@ generation_outputs gpttype_generate(const generation_inputs inputs, generation_o
{ {
if(embd.size()>1) if(embd.size()>1)
{ {
evalres = rwkv_eval_sequence(rwkv_ctx_v3, params.n_threads, (uint32_t*)embd.data(), embd.size(), rwkv_ctx_v3->state_in, rwkv_ctx_v3->state_out, rwkv_ctx_v3->logits_out); evalres = rwkv_eval_sequence(rwkv_ctx_v3, kcpp_params->n_threads, (uint32_t*)embd.data(), embd.size(), rwkv_ctx_v3->state_in, rwkv_ctx_v3->state_out, rwkv_ctx_v3->logits_out);
} }
else else
{ {
bool ignoreLogits = (!startedsampling && ((int)embd_inp.size() > input_consumed + 2)); bool ignoreLogits = (!startedsampling && ((int)embd_inp.size() > input_consumed + 2));
evalres = rwkv_eval(rwkv_ctx_v3, params.n_threads, embd[0], rwkv_ctx_v3->state_in, rwkv_ctx_v3->state_out, ignoreLogits?nullptr:rwkv_ctx_v3->logits_out); evalres = rwkv_eval(rwkv_ctx_v3, kcpp_params->n_threads, embd[0], rwkv_ctx_v3->state_in, rwkv_ctx_v3->state_out, ignoreLogits?nullptr:rwkv_ctx_v3->logits_out);
} }
memcpy(logits.data(), rwkv_ctx_v3->logits_out, sizeof(float) * rwkv_vocab.size()); memcpy(logits.data(), rwkv_ctx_v3->logits_out, sizeof(float) * rwkv_vocab.size());
@ -1802,39 +1803,39 @@ generation_outputs gpttype_generate(const generation_inputs inputs, generation_o
} }
else if(file_format==FileFormat::GPT2_1) else if(file_format==FileFormat::GPT2_1)
{ {
evalres = legacy_gpt2_eval(gpt2_ctx_v1, params.n_threads, n_past, embd, logits, mem_per_token, file_format); evalres = legacy_gpt2_eval(gpt2_ctx_v1, kcpp_params->n_threads, n_past, embd, logits, mem_per_token, file_format);
} }
else if(file_format==FileFormat::GPT2_2 || file_format==FileFormat::GPT2_3) else if(file_format==FileFormat::GPT2_2 || file_format==FileFormat::GPT2_3)
{ {
evalres = gpt2_v2_eval(gpt2_ctx_v2, params.n_threads, n_past, embd, logits, mem_per_token, file_format); evalres = gpt2_v2_eval(gpt2_ctx_v2, kcpp_params->n_threads, n_past, embd, logits, mem_per_token, file_format);
} }
else if(file_format==FileFormat::GPT2_4) else if(file_format==FileFormat::GPT2_4)
{ {
evalres = gpt2_eval(gpt2_ctx_v3, params.n_threads, n_past, embd, logits, mem_per_token, use_scratch); evalres = gpt2_eval(gpt2_ctx_v3, kcpp_params->n_threads, n_past, embd, logits, mem_per_token, use_scratch);
} }
else if(file_format==FileFormat::NEOX_1 || file_format == FileFormat::NEOX_2 || file_format == FileFormat::NEOX_3 || file_format==FileFormat::NEOX_4 || file_format==FileFormat::NEOX_5) else if(file_format==FileFormat::NEOX_1 || file_format == FileFormat::NEOX_2 || file_format == FileFormat::NEOX_3 || file_format==FileFormat::NEOX_4 || file_format==FileFormat::NEOX_5)
{ {
evalres = gpt_neox_v2_eval(neox_ctx_v2, params.n_threads, n_past, embd, logits, mem_per_token); evalres = gpt_neox_v2_eval(neox_ctx_v2, kcpp_params->n_threads, n_past, embd, logits, mem_per_token);
} }
else if(file_format==FileFormat::NEOX_6|| file_format==FileFormat::NEOX_7) else if(file_format==FileFormat::NEOX_6|| file_format==FileFormat::NEOX_7)
{ {
evalres = gpt_neox_eval(neox_ctx_v3, params.n_threads, n_past, embd, logits, mem_per_token, use_scratch); evalres = gpt_neox_eval(neox_ctx_v3, kcpp_params->n_threads, n_past, embd, logits, mem_per_token, use_scratch);
} }
else if(file_format==FileFormat::GPTJ_1 || file_format==FileFormat::GPTJ_2) else if(file_format==FileFormat::GPTJ_1 || file_format==FileFormat::GPTJ_2)
{ {
evalres = legacy_gptj_eval(gptj_ctx_v1, params.n_threads, n_past, embd, logits, mem_per_token, file_format); evalres = legacy_gptj_eval(gptj_ctx_v1, kcpp_params->n_threads, n_past, embd, logits, mem_per_token, file_format);
} }
else if(file_format==FileFormat::GPTJ_3 || file_format==FileFormat::GPTJ_4) else if(file_format==FileFormat::GPTJ_3 || file_format==FileFormat::GPTJ_4)
{ {
evalres = gptj_v2_eval(gptj_ctx_v2, params.n_threads, n_past, embd, logits, mem_per_token); evalres = gptj_v2_eval(gptj_ctx_v2, kcpp_params->n_threads, n_past, embd, logits, mem_per_token);
} }
else if(file_format==FileFormat::GPTJ_5) else if(file_format==FileFormat::GPTJ_5)
{ {
evalres = gptj_eval(gptj_ctx_v3, params.n_threads, n_past, embd, logits, mem_per_token, use_scratch); evalres = gptj_eval(gptj_ctx_v3, kcpp_params->n_threads, n_past, embd, logits, mem_per_token, use_scratch);
} }
else if(file_format==FileFormat::MPT_1) else if(file_format==FileFormat::MPT_1)
{ {
evalres = mpt_eval(mpt_ctx_v3, params.n_threads, n_past, embd, logits, false, mem_per_token, use_scratch); evalres = mpt_eval(mpt_ctx_v3, kcpp_params->n_threads, n_past, embd, logits, false, mem_per_token, use_scratch);
} }
else else
{ {
@ -1856,21 +1857,21 @@ generation_outputs gpttype_generate(const generation_inputs inputs, generation_o
if ((int)embd_inp.size() <= input_consumed) if ((int)embd_inp.size() <= input_consumed)
{ {
// out of user input, sample next token // out of user input, sample next token
const float top_k = params.top_k; const float top_k = kcpp_params->top_k;
const float top_p = params.top_p; const float top_p = kcpp_params->top_p;
const float min_p = params.min_p; const float min_p = kcpp_params->min_p;
const float temp = params.temp; const float temp = kcpp_params->temp;
const float top_a = inputs.top_a; const float top_a = inputs.top_a;
const float repeat_penalty = params.repeat_penalty; const float repeat_penalty = kcpp_params->repeat_penalty;
const float presence_penalty = params.presence_penalty; const float presence_penalty = kcpp_params->presence_penalty;
const float typical_p = params.typical_p; const float typical_p = kcpp_params->typical_p;
const float tfs_z = params.tfs_z; const float tfs_z = kcpp_params->tfs_z;
if (!startedsampling) if (!startedsampling)
{ {
startedsampling = true; startedsampling = true;
params.n_batch = original_batch; kcpp_params->n_batch = original_batch;
params.n_threads = original_threads; kcpp_params->n_threads = original_threads;
time1 = timer_check(); time1 = timer_check();
timer_start(); timer_start();
if(allow_regular_prints) if(allow_regular_prints)
@ -1920,7 +1921,7 @@ generation_outputs gpttype_generate(const generation_inputs inputs, generation_o
id = SampleLogits(logitsPtr, nctx, n_vocab, last_n_size, repeat_penalty, presence_penalty, id = SampleLogits(logitsPtr, nctx, n_vocab, last_n_size, repeat_penalty, presence_penalty,
top_k, top_a, top_p, min_p, typical_p, tfs_z, temp, rng, top_k, top_a, top_p, min_p, typical_p, tfs_z, temp, rng,
params.mirostat, params.mirostat_tau, params.mirostat_eta, sampler_order, grammar); kcpp_params->mirostat, kcpp_params->mirostat_tau, kcpp_params->mirostat_eta, sampler_order, grammar);
if (grammar != nullptr) { if (grammar != nullptr) {
grammar_accept_token(file_format, n_vocab, grammar, id); grammar_accept_token(file_format, n_vocab, grammar, id);
@ -1950,7 +1951,7 @@ generation_outputs gpttype_generate(const generation_inputs inputs, generation_o
if (startedsampling && allow_regular_prints) if (startedsampling && allow_regular_prints)
{ {
printf("\rGenerating (%d / %d tokens)", (params.n_predict - remaining_tokens), params.n_predict); printf("\rGenerating (%d / %d tokens)", (kcpp_params->n_predict - remaining_tokens), kcpp_params->n_predict);
} }
if(debugmode==1 && top_picks.size()>0) if(debugmode==1 && top_picks.size()>0)
{ {
@ -2009,7 +2010,7 @@ generation_outputs gpttype_generate(const generation_inputs inputs, generation_o
last_n_tokens.push_back(embd_inp[input_consumed]); last_n_tokens.push_back(embd_inp[input_consumed]);
current_context_tokens.push_back(embd_inp[input_consumed]); current_context_tokens.push_back(embd_inp[input_consumed]);
++input_consumed; ++input_consumed;
if ((int)embd.size() >= params.n_batch) if ((int)embd.size() >= kcpp_params->n_batch)
{ {
break; break;
} }
@ -2018,7 +2019,7 @@ generation_outputs gpttype_generate(const generation_inputs inputs, generation_o
} }
time2 = timer_check(); time2 = timer_check();
float pt1 = (time1*1000.0/(embd_inp.size()==0?1:embd_inp.size())); float pt1 = (time1*1000.0/(embd_inp.size()==0?1:embd_inp.size()));
int realnpredict = params.n_predict-stopper_unused_tokens; int realnpredict = kcpp_params->n_predict-stopper_unused_tokens;
float pt2 = (time2*1000.0/(realnpredict==0?1:realnpredict)); float pt2 = (time2*1000.0/(realnpredict==0?1:realnpredict));
float tokens_per_second = (realnpredict == 0 ? 0 : realnpredict / (time1 + time2)); float tokens_per_second = (realnpredict == 0 ? 0 : realnpredict / (time1 + time2));
printf("\nContextLimit: %d/%d, Processing:%.2fs (%.1fms/T), Generation:%.2fs (%.1fms/T), Total:%.2fs (%.2fT/s)",current_context_tokens.size(),nctx, time1, pt1, time2, pt2, (time1 + time2), tokens_per_second); printf("\nContextLimit: %d/%d, Processing:%.2fs (%.1fms/T), Generation:%.2fs (%.1fms/T), Total:%.2fs (%.2fT/s)",current_context_tokens.size(),nctx, time1, pt1, time2, pt2, (time1 + time2), tokens_per_second);

20
klite.embd
View file

@ -3349,6 +3349,7 @@ Current version: 102
gui_type_chat: 1, //0=standard, 1=messenger, 2=aesthetic gui_type_chat: 1, //0=standard, 1=messenger, 2=aesthetic
gui_type_instruct: 0, //0=standard, 1=messenger, 2=aesthetic gui_type_instruct: 0, //0=standard, 1=messenger, 2=aesthetic
multiline_replies: true, multiline_replies: true,
multiline_replies_adventure: true,
allow_continue_chat: false, allow_continue_chat: false,
idle_responses: 0, idle_responses: 0,
idle_duration: 60, idle_duration: 60,
@ -7209,6 +7210,7 @@ Current version: 102
toggle_generate_images_mode(true); toggle_generate_images_mode(true);
} }
document.getElementById("multiline_replies").checked = localsettings.multiline_replies; document.getElementById("multiline_replies").checked = localsettings.multiline_replies;
document.getElementById("multiline_replies_adventure").checked = localsettings.multiline_replies_adventure;
document.getElementById("allow_continue_chat").checked = localsettings.allow_continue_chat; document.getElementById("allow_continue_chat").checked = localsettings.allow_continue_chat;
document.getElementById("idle_responses").value = localsettings.idle_responses; document.getElementById("idle_responses").value = localsettings.idle_responses;
document.getElementById("idle_duration").value = localsettings.idle_duration; document.getElementById("idle_duration").value = localsettings.idle_duration;
@ -7391,6 +7393,7 @@ Current version: 102
localsettings.gui_type_instruct = document.getElementById("gui_type").value; localsettings.gui_type_instruct = document.getElementById("gui_type").value;
} }
localsettings.multiline_replies = (document.getElementById("multiline_replies").checked ? true : false); localsettings.multiline_replies = (document.getElementById("multiline_replies").checked ? true : false);
localsettings.multiline_replies_adventure = (document.getElementById("multiline_replies_adventure").checked ? true : false);
localsettings.allow_continue_chat = (document.getElementById("allow_continue_chat").checked ? true : false); localsettings.allow_continue_chat = (document.getElementById("allow_continue_chat").checked ? true : false);
localsettings.idle_responses = document.getElementById("idle_responses").value; localsettings.idle_responses = document.getElementById("idle_responses").value;
localsettings.idle_duration = document.getElementById("idle_duration").value; localsettings.idle_duration = document.getElementById("idle_duration").value;
@ -8639,6 +8642,10 @@ Current version: 102
if (localsettings.opmode == 2) //stop on new action found if (localsettings.opmode == 2) //stop on new action found
{ {
seqs = ["\n\> "]; seqs = ["\n\> "];
if(!localsettings.multiline_replies_adventure)
{
seqs.push("\n");
}
} }
if (localsettings.opmode == 3) //stop on selfname found if (localsettings.opmode == 3) //stop on selfname found
{ {
@ -9436,6 +9443,15 @@ Current version: 102
splitresponse = gentxt.split("\n\> "); splitresponse = gentxt.split("\n\> ");
gentxt = splitresponse[0]; gentxt = splitresponse[0];
} }
if(!localsettings.multiline_replies_adventure)
{
let foundnl = gentxt.indexOf("\n");
if (foundnl != -1) //if found, truncate to it
{
splitresponse = gentxt.split("\n");
gentxt = splitresponse[0];
}
}
} }
//if we are in chatmode, truncate to my first response //if we are in chatmode, truncate to my first response
@ -12285,6 +12301,10 @@ Current version: 102
</div> </div>
</div> </div>
<div id="adventuresection2" class="settinglabel hidden" style="padding-top: 3px;"> <div id="adventuresection2" class="settinglabel hidden" style="padding-top: 3px;">
<div class="settinglabel">
<div class="justifyleft settingsmall" title="Whether to allow multiple lines in AI responses.">Multiline Replies </div>
<input type="checkbox" id="multiline_replies_adventure" style="margin:0px 0 0;">
</div>
</div> </div>
<div id="instructsection2" class="settinglabel hidden" style="padding-top: 3px;"> <div id="instructsection2" class="settinglabel hidden" style="padding-top: 3px;">
<div class="justifyleft settingsmall">Instruct Tag Preset <span class="helpicon">?<span class="helptext">Quickly select between common instruct tag formats. Different models are trained with different tags.</span></span></div> <div class="justifyleft settingsmall">Instruct Tag Preset <span class="helpicon">?<span class="helptext">Quickly select between common instruct tag formats. Different models are trained with different tags.</span></span></div>