vbatts/llama.cpp
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

llama.cpp : fix bpe tokenizer

Browse source
This commit is contained in:
klosax 2023-08-23 00:02:13 +02:00 • committed by GitHub
parent ffa5099c6d
commit a95ae7526a
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
1 changed files with 8 additions and 4 deletions
Download patch file Download diff file Expand all files Collapse all files

12
llama.cpp
View file

@ -2839,10 +2839,14 @@ static bool llama_is_pad_token(const llama_vocab & vocab, llama_token id ) {
} }
static uint8_t llama_token_to_byte(const llama_vocab & vocab, llama_token id) { static uint8_t llama_token_to_byte(const llama_vocab & vocab, llama_token id) {
GGML_ASSERT(llama_is_byte_token(vocab, id)); if (vocab.type == LLAMA_VOCAB_TYPE_SPM) {
const auto& token_data = vocab.id_to_token.at(id); char buf[7];
auto buf = token_data.text.substr(3, 2); int result = snprintf(buf, sizeof(buf), "<0x%02X>", ch);
return strtol(buf.c_str(), NULL, 16); GGML_ASSERT(0 <= result && result < 7);
return vocab.token_to_id.at(buf);
}
// vocab.type == LLAMA_VOCAB_TYPE_BPE
return vocab.token_to_id.at(std::string(1, ch));
} }
static llama_token llama_byte_to_token(const llama_vocab & vocab, uint8_t ch) { static llama_token llama_byte_to_token(const llama_vocab & vocab, uint8_t ch) {