From f8ee54bd2c09aad29bcfd3b619e66e5979717836 Mon Sep 17 00:00:00 2001 From: Georgi Gerganov Date: Wed, 23 Aug 2023 20:39:24 +0300 Subject: [PATCH] llama : revert BPE special-case in llama_byte_to_token() --- llama.cpp | 12 ++++-------- 1 file changed, 4 insertions(+), 8 deletions(-) diff --git a/llama.cpp b/llama.cpp index 58b7b70d3..d3adda11b 100644 --- a/llama.cpp +++ b/llama.cpp @@ -2971,14 +2971,10 @@ static uint8_t llama_token_to_byte(const llama_vocab & vocab, llama_token id) { } static llama_token llama_byte_to_token(const llama_vocab & vocab, uint8_t ch) { - if (vocab.type == LLAMA_VOCAB_TYPE_SPM) { - char buf[7]; - int result = snprintf(buf, sizeof(buf), "<0x%02X>", ch); - GGML_ASSERT(0 <= result && result < 7); - return vocab.token_to_id.at(buf); - } - // vocab.type == LLAMA_VOCAB_TYPE_BPE - return vocab.token_to_id.at(std::string(1, ch)); + char buf[7]; + int result = snprintf(buf, sizeof(buf), "<0x%02X>", ch); + GGML_ASSERT(0 <= result && result < 7); + return vocab.token_to_id.at(buf); } static std::string llama_escape_whitespace(const std::string& text) {