vbatts/llama.cpp
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

llama.cpp : fix the fix of bpe tokenizer

Browse source
This commit is contained in:
klosax 2023-08-23 00:06:53 +02:00 • committed by GitHub
parent a95ae7526a
commit d561b7f724
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
1 changed files with 7 additions and 7 deletions
Download patch file Download diff file Expand all files Collapse all files

14
llama.cpp
View file

@ -2839,6 +2839,13 @@ static bool llama_is_pad_token(const llama_vocab & vocab, llama_token id ) {
} }
static uint8_t llama_token_to_byte(const llama_vocab & vocab, llama_token id) { static uint8_t llama_token_to_byte(const llama_vocab & vocab, llama_token id) {
GGML_ASSERT(llama_is_byte_token(vocab, id));
const auto& token_data = vocab.id_to_token.at(id);
auto buf = token_data.text.substr(3, 2);
return strtol(buf.c_str(), NULL, 16);
}
static llama_token llama_byte_to_token(const llama_vocab & vocab, uint8_t ch) {
if (vocab.type == LLAMA_VOCAB_TYPE_SPM) { if (vocab.type == LLAMA_VOCAB_TYPE_SPM) {
char buf[7]; char buf[7];
int result = snprintf(buf, sizeof(buf), "<0x%02X>", ch); int result = snprintf(buf, sizeof(buf), "<0x%02X>", ch);
@ -2849,13 +2856,6 @@ static uint8_t llama_token_to_byte(const llama_vocab & vocab, llama_token id) {
return vocab.token_to_id.at(std::string(1, ch)); return vocab.token_to_id.at(std::string(1, ch));
} }
static llama_token llama_byte_to_token(const llama_vocab & vocab, uint8_t ch) {
char buf[7];
int result = snprintf(buf, sizeof(buf), "<0x%02X>", ch);
GGML_ASSERT(0 <= result && result < 7);
return vocab.token_to_id.at(buf);
}
static std::string llama_escape_whitespace(const std::string& text) { static std::string llama_escape_whitespace(const std::string& text) {
std::string result; std::string result;
bool escaping = false; bool escaping = false;