vbatts/llama.cpp
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

llama : revert BPE special-case in llama_byte_to_token()

Browse source
This commit is contained in:
Georgi Gerganov 2023-08-23 20:39:24 +03:00
parent 596e1094fb
commit f8ee54bd2c
No known key found for this signature in database
GPG key ID: 449E073F9DC10735
1 changed files with 4 additions and 8 deletions
Download patch file Download diff file Expand all files Collapse all files

12
llama.cpp
View file

@ -2971,14 +2971,10 @@ static uint8_t llama_token_to_byte(const llama_vocab & vocab, llama_token id) {
}
static llama_token llama_byte_to_token(const llama_vocab & vocab, uint8_t ch) {
if (vocab.type == LLAMA_VOCAB_TYPE_SPM) {
char buf[7];
int result = snprintf(buf, sizeof(buf), "<0x%02X>", ch);
GGML_ASSERT(0 <= result && result < 7);
return vocab.token_to_id.at(buf);
}
// vocab.type == LLAMA_VOCAB_TYPE_BPE
return vocab.token_to_id.at(std::string(1, ch));
char buf[7];
int result = snprintf(buf, sizeof(buf), "<0x%02X>", ch);
GGML_ASSERT(0 <= result && result < 7);
return vocab.token_to_id.at(buf);
}
static std::string llama_escape_whitespace(const std::string& text) {