From c3a473d421f9ccbada569a6f728145f50269b131 Mon Sep 17 00:00:00 2001 From: Georgi Gerganov Date: Mon, 6 Jan 2025 10:54:11 +0200 Subject: [PATCH] tokenize : update help --- examples/tokenize/tokenize.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/examples/tokenize/tokenize.cpp b/examples/tokenize/tokenize.cpp index 76624c8b0..57d9d4312 100644 --- a/examples/tokenize/tokenize.cpp +++ b/examples/tokenize/tokenize.cpp @@ -31,6 +31,7 @@ static void print_usage_information(const char * argv0) { printf(" -p PROMPT, --prompt PROMPT read prompt from the argument.\n"); printf(" --stdin read prompt from standard input.\n"); printf(" --no-bos do not ever add a BOS token to the prompt, even if normally the model uses a BOS token.\n"); + printf(" --no-escape do not escape input (such as \\n, \\t, etc.).\n"); printf(" --no-parse-special do not parse control tokens.\n"); printf(" --log-disable disable logs. Makes stderr quiet when loading the model.\n"); printf(" --show-count print the total number of tokens.\n");