vbatts/llama.cpp
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

tokenize : escape the prompt

Browse source
This commit is contained in:
Georgi Gerganov 2025-01-03 12:05:16 +02:00
parent e7da954ecc
commit fc49c3230a
No known key found for this signature in database
GPG key ID: 449E073F9DC10735
1 changed files with 9 additions and 0 deletions
Download patch file Download diff file Expand all files Collapse all files

9
examples/tokenize/tokenize.cpp
View file

@ -198,6 +198,7 @@ int main(int raw_argc, char ** raw_argv) {
// variables where to put any arguments we see. // variables where to put any arguments we see.
bool printing_ids = false; bool printing_ids = false;
bool no_bos = false; bool no_bos = false;
bool no_escape = false;
bool no_parse_special = false; bool no_parse_special = false;
bool disable_logging = false; bool disable_logging = false;
bool show_token_count = false; bool show_token_count = false;
@ -233,6 +234,9 @@ int main(int raw_argc, char ** raw_argv) {
else if (arg == "--no-bos") { else if (arg == "--no-bos") {
no_bos = true; no_bos = true;
} }
else if (arg == "--no-escape") {
no_escape = true;
}
else if (arg == "--no-parse-special") { else if (arg == "--no-parse-special") {
no_parse_special = true; no_parse_special = true;
} }
@ -363,6 +367,11 @@ int main(int raw_argc, char ** raw_argv) {
const bool model_wants_add_bos = llama_add_bos_token(model); const bool model_wants_add_bos = llama_add_bos_token(model);
const bool add_bos = model_wants_add_bos && !no_bos; const bool add_bos = model_wants_add_bos && !no_bos;
const bool parse_special = !no_parse_special; const bool parse_special = !no_parse_special;
const bool escape = !no_escape;
if (escape) {
string_process_escapes(prompt);
}
std::vector<llama_token> tokens; std::vector<llama_token> tokens;
tokens = common_tokenize(model, prompt, add_bos, parse_special); tokens = common_tokenize(model, prompt, add_bos, parse_special);