vbatts/llama.cpp
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

tokenize : update help

Browse source
This commit is contained in:
Georgi Gerganov 2025-01-06 10:54:11 +02:00
parent fc49c3230a
commit c3a473d421
No known key found for this signature in database
GPG key ID: 449E073F9DC10735
1 changed files with 1 additions and 0 deletions
Download patch file Download diff file Expand all files Collapse all files

1
examples/tokenize/tokenize.cpp
View file

@ -31,6 +31,7 @@ static void print_usage_information(const char * argv0) {
printf(" -p PROMPT, --prompt PROMPT read prompt from the argument.\n");
printf(" --stdin read prompt from standard input.\n");
printf(" --no-bos do not ever add a BOS token to the prompt, even if normally the model uses a BOS token.\n");
printf(" --no-escape do not escape input (such as \\n, \\t, etc.).\n");
printf(" --no-parse-special do not parse control tokens.\n");
printf(" --log-disable disable logs. Makes stderr quiet when loading the model.\n");
printf(" --show-count print the total number of tokens.\n");