vbatts/llama.cpp
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

fix: revert showing control tokens by default

Browse source
This commit is contained in:
Kyle Mistele 2024-04-23 22:25:46 -05:00
parent 4e96a812b3
commit 572960a045
1 changed files with 2 additions and 2 deletions
Download patch file Download diff file Expand all files Collapse all files

4
common/common.cpp
View file

@ -2328,10 +2328,10 @@ std::vector<llama_token> llama_tokenize(
std::string llama_token_to_piece(const struct llama_context * ctx, llama_token token) {
std::vector<char> result(8, 0);
const int n_tokens = llama_token_to_piece(llama_get_model(ctx), token, result.data(), result.size(), true);
const int n_tokens = llama_token_to_piece(llama_get_model(ctx), token, result.data(), result.size(), false);
if (n_tokens < 0) {
result.resize(-n_tokens);
int check = llama_token_to_piece(llama_get_model(ctx), token, result.data(), result.size(), true);
int check = llama_token_to_piece(llama_get_model(ctx), token, result.data(), result.size(), false);
GGML_ASSERT(check == -n_tokens);
} else {
result.resize(n_tokens);