vbatts/llama.cpp
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Hotfix prompt caching introduced in #1169, fixes #1257

Browse source
This commit is contained in:
Ivan Stepanov 2023-05-01 03:34:34 +03:00
parent f0d70f147d
commit c9fdebc02c
2 changed files with 4 additions and 3 deletions
Download patch file Download diff file Expand all files Collapse all files

2
examples/main/main.cpp
View file

@ -404,7 +404,7 @@ int main(int argc, char ** argv) {
// optionally save the session on first sample (for faster prompt loading next time) // optionally save the session on first sample (for faster prompt loading next time)
if (!path_session.empty() && need_to_save_session) { if (!path_session.empty() && need_to_save_session) {
need_to_save_session = false; need_to_save_session = false;
llama_save_session_file(ctx, path_session.c_str(), session_tokens.data(), session_tokens.size()); llama_save_session_file(ctx, path_session.c_str(), session_tokens.data(), session_tokens.size() - 1); // FIXME: -1 is a hack
} }
llama_token id = 0; llama_token id = 0;

5
llama.cpp
View file

@ -2701,7 +2701,7 @@ size_t llama_load_session_file(struct llama_context * ctx, const char * path_ses
const uint32_t magic = file.read_u32(); const uint32_t magic = file.read_u32();
const uint32_t version = file.read_u32(); const uint32_t version = file.read_u32();
if (!(magic == 'ggsn' && version == 0)) { if (!(magic == 'ggsn' && version == 1)) {
fprintf(stderr, "%s : unknown (magic, version) for session file: %08x, %08x\n", __func__, magic, version); fprintf(stderr, "%s : unknown (magic, version) for session file: %08x, %08x\n", __func__, magic, version);
return 0; return 0;
} }
@ -2724,6 +2724,7 @@ size_t llama_load_session_file(struct llama_context * ctx, const char * path_ses
const size_t n_orig_state_size = llama_get_state_size(ctx); const size_t n_orig_state_size = llama_get_state_size(ctx);
if (n_state_size != n_orig_state_size) { if (n_state_size != n_orig_state_size) {
fprintf(stderr, "%s : failed to validate state size\n", __func__); fprintf(stderr, "%s : failed to validate state size\n", __func__);
return 0;
} }
std::unique_ptr<uint8_t[]> state_data(new uint8_t[n_state_size]); std::unique_ptr<uint8_t[]> state_data(new uint8_t[n_state_size]);
file.read_raw(state_data.get(), n_state_size); file.read_raw(state_data.get(), n_state_size);
@ -2739,7 +2740,7 @@ size_t llama_save_session_file(struct llama_context * ctx, const char * path_ses
llama_copy_state_data(ctx, state_data.get()); llama_copy_state_data(ctx, state_data.get());
file.write_u32('ggsn'); // magic file.write_u32('ggsn'); // magic
file.write_u32(0); // version file.write_u32(1); // version
file.write_raw(&ctx->model.hparams, sizeof(llama_hparams)); file.write_raw(&ctx->model.hparams, sizeof(llama_hparams));
file.write_u32((uint32_t) n_token_count); // REVIEW file.write_u32((uint32_t) n_token_count); // REVIEW