vbatts/llama.cpp
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

llama : fix bpe tokenize from byte

Browse source
This commit is contained in:
opparco 2023-08-30 11:28:19 +09:00
parent 06abf8eeba
commit 52f8cc8f37
1 changed files with 8 additions and 2 deletions
Download patch file Download diff file Expand all files Collapse all files

6
llama.cpp
View file

@ -3319,10 +3319,16 @@ struct llm_tokenizer_bpe {
std::string byte_str(1, *j); std::string byte_str(1, *j);
auto token_multibyte = vocab.token_to_id.find(byte_str); auto token_multibyte = vocab.token_to_id.find(byte_str);
if (token_multibyte == vocab.token_to_id.end()) { if (token_multibyte == vocab.token_to_id.end()) {
try {
llama_token token_byte = llama_byte_to_token(vocab, *j);
output.push_back(token_byte);
} catch (const std::out_of_range & err) {
fprintf(stderr,"ERROR: byte not found in vocab: '%s'\n", byte_str.c_str()); fprintf(stderr,"ERROR: byte not found in vocab: '%s'\n", byte_str.c_str());
} }
} else {
output.push_back((*token_multibyte).second); output.push_back((*token_multibyte).second);
} }
}
} else { } else {
output.push_back((*token).second); output.push_back((*token).second);
} }