vbatts/llama.cpp
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

main : don't print special tokens with --grammar

Browse source 
The CLI interface was recently changed to print special control tokens
like the </s> stop message one. This token shouldn't be printed if the
grammar flag was passed, unless the grammar specifies it, because that
breaks shell-scriptability.
This commit is contained in:
Justine Tunney 2024-04-26 02:01:58 -07:00
parent 83b72cb086
commit 08c5e35014
No known key found for this signature in database
GPG key ID: 52965314629936D4
1 changed files with 4 additions and 2 deletions
Download patch file Download diff file Expand all files Collapse all files

6
examples/main/main.cpp
View file

@ -520,6 +520,7 @@ int main(int argc, char ** argv) {
}
struct llama_sampling_context * ctx_sampling = llama_sampling_init(sparams);
bool should_show_special_tokens = sparams.grammar.empty();
while ((n_remain != 0 && !is_antiprompt) || params.interactive) {
// predict
@ -733,7 +734,8 @@ int main(int argc, char ** argv) {
// display text
if (input_echo && display) {
for (auto id : embd) {
const std::string token_str = llama_token_to_piece(ctx, id);
const std::string token_str =
llama_token_to_piece(ctx, id, should_show_special_tokens);
printf("%s", token_str.c_str());
if (embd.size() > 1) {
@ -899,7 +901,7 @@ int main(int argc, char ** argv) {
for (size_t i = original_size; i < embd_inp.size(); ++i) {
const llama_token token = embd_inp[i];
output_tokens.push_back(token);
output_ss << llama_token_to_piece(ctx, token);
output_ss << llama_token_to_piece(ctx, token, should_show_special_tokens);
}
n_remain -= line_inp.size();