vbatts/llama.cpp
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

llama : fix detection of control-like user-defined tokens

Browse source
This commit is contained in:
Francis Couture-Harpin 2024-07-07 16:13:35 -04:00
parent 6b961e3d24
commit 56df1fcdcb
2 changed files with 4 additions and 3 deletions
Download patch file Download diff file Expand all files Collapse all files

3
src/llama.cpp
View file

@ -5513,7 +5513,8 @@ static void llm_load_vocab(
} }
} }
if ((token_data.attr & LLAMA_TOKEN_ATTR_USER_DEFINED) && token_data.text.find('<') && token_data.text.rfind('>')) { if ((token_data.attr & LLAMA_TOKEN_ATTR_USER_DEFINED) && !token_data.text.empty() &&
token_data.text.front() == '<' && token_data.text.back() == '>') {
// Some models mark some added tokens which ought to be control tokens as not special. // Some models mark some added tokens which ought to be control tokens as not special.
// (e.g. command-r, command-r-plus, deepseek-coder) // (e.g. command-r, command-r-plus, deepseek-coder)
// TODO: should this be fixed in the convert script instead? // TODO: should this be fixed in the convert script instead?

4
tests/test-tokenizer-0.cpp
View file

@ -195,7 +195,7 @@ int main(int argc, char **argv) {
const bool add_special = false; const bool add_special = false;
for (const auto & test_kv : k_tests) { for (const auto & test_kv : k_tests) {
const std::vector<llama_token> res = llama_tokenize(ctx, test_kv.first, add_special); const std::vector<llama_token> res = llama_tokenize(ctx, test_kv.first, add_special, false);
printf("\n"); printf("\n");
printf("src: '%s'\n", test_kv.first.c_str()); printf("src: '%s'\n", test_kv.first.c_str());
@ -253,7 +253,7 @@ int main(int argc, char **argv) {
{ {
const auto t_start = ggml_time_us(); const auto t_start = ggml_time_us();
res = llama_tokenize(ctx, text, add_special); res = llama_tokenize(ctx, text, add_special, false);
const auto t_end = ggml_time_us(); const auto t_end = ggml_time_us();