vbatts/llama.cpp
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

PR comments

Browse source
This commit is contained in:
Evan Jones 2023-05-08 23:10:51 -04:00
parent e4429e912b
commit 8c88b172c5
3 changed files with 16 additions and 16 deletions
Download patch file Download diff file Expand all files Collapse all files

5
examples/common.cpp
View file

@ -125,7 +125,7 @@ bool gpt_params_parse(int argc, char ** argv, gpt_params & params) {
} }
params.path_prompt_cache = argv[i]; params.path_prompt_cache = argv[i];
} else if (arg == "--prompt-cache-all") { } else if (arg == "--prompt-cache-all") {
params.prompt_cache_save_all = true; params.prompt_cache_all = true;
} else if (arg == "-f" || arg == "--file") { } else if (arg == "-f" || arg == "--file") {
if (++i >= argc) { if (++i >= argc) {
invalid_param = true; invalid_param = true;
@ -370,7 +370,8 @@ void gpt_print_usage(int /*argc*/, char ** argv, const gpt_params & params) {
fprintf(stderr, " prompt to start generation with (default: empty)\n"); fprintf(stderr, " prompt to start generation with (default: empty)\n");
fprintf(stderr, " -e process prompt escapes sequences (\\n, \\r, \\t, \\', \\\", \\\\)\n"); fprintf(stderr, " -e process prompt escapes sequences (\\n, \\r, \\t, \\', \\\", \\\\)\n");
fprintf(stderr, " --prompt-cache FNAME file to cache prompt state for faster startup (default: none)\n"); fprintf(stderr, " --prompt-cache FNAME file to cache prompt state for faster startup (default: none)\n");
fprintf(stderr, " --prompt-cache-all if specified, saves user input and generations to cache as well\n"); fprintf(stderr, " --prompt-cache-all if specified, saves user input and generations to cache as well.\n");
fprintf(stderr, " not supported with --interactive or other interactive options\n");
fprintf(stderr, " --random-prompt start with a randomized prompt.\n"); fprintf(stderr, " --random-prompt start with a randomized prompt.\n");
fprintf(stderr, " --in-prefix STRING string to prefix user inputs with (default: empty)\n"); fprintf(stderr, " --in-prefix STRING string to prefix user inputs with (default: empty)\n");
fprintf(stderr, " --in-suffix STRING string to suffix after user inputs with (default: empty)\n"); fprintf(stderr, " --in-suffix STRING string to suffix after user inputs with (default: empty)\n");

10
examples/common.h
View file

@ -54,11 +54,11 @@ struct gpt_params {
std::string lora_adapter = ""; // lora adapter path std::string lora_adapter = ""; // lora adapter path
std::string lora_base = ""; // base model path for the lora adapter std::string lora_base = ""; // base model path for the lora adapter
bool memory_f16 = true; // use f16 instead of f32 for memory kv bool memory_f16 = true; // use f16 instead of f32 for memory kv
bool random_prompt = false; // do not randomize prompt if none provided bool random_prompt = false; // do not randomize prompt if none provided
bool use_color = false; // use color to distinguish generations and inputs bool use_color = false; // use color to distinguish generations and inputs
bool interactive = false; // interactive mode bool interactive = false; // interactive mode
bool prompt_cache_save_all = false; // save user input and generations to prompt cache bool prompt_cache_all = false; // save user input and generations to prompt cache
bool embedding = false; // get only sentence embedding bool embedding = false; // get only sentence embedding
bool interactive_first = false; // wait for user input immediately bool interactive_first = false; // wait for user input immediately

17
examples/main/main.cpp
View file

@ -140,7 +140,6 @@ int main(int argc, char ** argv) {
params.prompt.insert(0, 1, ' '); params.prompt.insert(0, 1, ' ');
std::string path_session = params.path_prompt_cache; std::string path_session = params.path_prompt_cache;
const bool session_save_all = params.prompt_cache_save_all;
std::vector<llama_token> session_tokens; std::vector<llama_token> session_tokens;
if (!path_session.empty()) { if (!path_session.empty()) {
@ -236,6 +235,11 @@ int main(int argc, char ** argv) {
} }
if (params.interactive) { if (params.interactive) {
if (params.prompt_cache_all) {
fprintf(stderr, "error: --prompt-cache-all not supported in interactive mode yet\n");
return 1;
}
#if defined (__unix__) || (defined (__APPLE__) && defined (__MACH__)) #if defined (__unix__) || (defined (__APPLE__) && defined (__MACH__))
struct sigaction sigint_action; struct sigaction sigint_action;
sigint_action.sa_handler = sigint_handler; sigint_action.sa_handler = sigint_handler;
@ -295,8 +299,7 @@ int main(int argc, char ** argv) {
bool is_antiprompt = false; bool is_antiprompt = false;
bool input_echo = true; bool input_echo = true;
bool need_to_save_session = !path_session.empty(); bool need_to_save_session = !path_session.empty() && n_matching_session_tokens < embd_inp.size();
int n_past = 0; int n_past = 0;
int n_remain = params.n_predict; int n_remain = params.n_predict;
@ -325,11 +328,7 @@ int main(int argc, char ** argv) {
embd.insert(embd.begin(), last_n_tokens.begin() + n_ctx - n_left/2 - embd.size(), last_n_tokens.end() - embd.size()); embd.insert(embd.begin(), last_n_tokens.begin() + n_ctx - n_left/2 - embd.size(), last_n_tokens.end() - embd.size());
// stop saving session if we run out of context // stop saving session if we run out of context
if (!path_session.empty() && session_save_all) { path_session.clear();
llama_save_session_file(ctx, path_session.c_str(),
session_tokens.data(), session_tokens.size());
}
path_session = "";
//printf("\n---\n"); //printf("\n---\n");
//printf("resetting: '"); //printf("resetting: '");
@ -604,7 +603,7 @@ int main(int argc, char ** argv) {
} }
} }
if (!path_session.empty() && session_save_all) { if (!path_session.empty() && params.prompt_cache_all) {
fprintf(stderr, "\n%s: saving final output to session file '%s'\n", __func__, path_session.c_str()); fprintf(stderr, "\n%s: saving final output to session file '%s'\n", __func__, path_session.c_str());
llama_save_session_file(ctx, path_session.c_str(), session_tokens.data(), session_tokens.size()); llama_save_session_file(ctx, path_session.c_str(), session_tokens.data(), session_tokens.size());
} }