vbatts/llama.cpp
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

tokenize newline token

Browse source
This commit is contained in:
rabidcopy 2023-03-22 19:19:49 -05:00 • committed by GitHub
parent 8f83ce8380
commit 7864eef92c
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
1 changed files with 4 additions and 1 deletions
Download patch file Download diff file Expand all files Collapse all files

5
main.cpp
View file

@ -254,6 +254,9 @@ int main(int argc, char ** argv) {
params.interactive = true; params.interactive = true;
} }
//determine newline token
auto llama_token_newline = ::llama_tokenize(ctx, "\n", false);
fprintf(stderr, "\n"); fprintf(stderr, "\n");
fprintf(stderr, "%s: prompt: '%s'\n", __func__, params.prompt.c_str()); fprintf(stderr, "%s: prompt: '%s'\n", __func__, params.prompt.c_str());
fprintf(stderr, "%s: number of tokens in prompt = %zu\n", __func__, embd_inp.size()); fprintf(stderr, "%s: number of tokens in prompt = %zu\n", __func__, embd_inp.size());
@ -357,7 +360,7 @@ int main(int argc, char ** argv) {
// replace end of text token with newline token when in interactive mode // replace end of text token with newline token when in interactive mode
if (id == llama_token_eos() && params.interactive) { if (id == llama_token_eos() && params.interactive) {
id = llama_token_newline(); id = llama_token_newline.front();
if (params.antiprompt.size() != 0) { if (params.antiprompt.size() != 0) {
// tokenize and inject first reverse prompt // tokenize and inject first reverse prompt
first_antiprompt = ::llama_tokenize(ctx, params.antiprompt.front(), false); first_antiprompt = ::llama_tokenize(ctx, params.antiprompt.front(), false);