vbatts/llama.cpp
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Guarding some unusable code pathes

Browse source
This commit is contained in:
goerch 2023-09-19 14:17:34 +02:00
parent 1b7c3692af
commit a4e9448ef0
2 changed files with 9 additions and 3 deletions
Download patch file Download diff file Expand all files Collapse all files

1
common/common.cpp
View file

@ -1,5 +1,4 @@
#include "common.h" #include "common.h"
#include "unicode.h"
#include "build-info.h" #include "build-info.h"
#include "llama.h" #include "llama.h"

11
llama.cpp
View file

@ -3896,6 +3896,8 @@ static uint8_t llama_token_to_byte(const llama_vocab& vocab, llama_token id) {
GGML_ASSERT(false); GGML_ASSERT(false);
return unicode_to_bytes_bpe(token_data.text); return unicode_to_bytes_bpe(token_data.text);
} }
default:
GGML_ASSERT(false);
} }
} }
@ -3910,6 +3912,8 @@ static llama_token llama_byte_to_token(const llama_vocab & vocab, uint8_t ch) {
case LLAMA_VOCAB_TYPE_BPE: { case LLAMA_VOCAB_TYPE_BPE: {
return vocab.token_to_id.at(bytes_to_unicode_bpe(ch)); return vocab.token_to_id.at(bytes_to_unicode_bpe(ch));
} }
default:
GGML_ASSERT(false);
} }
} }
@ -7226,6 +7230,8 @@ int llama_token_to_piece_with_model(const struct llama_model * model, llama_toke
llama_unescape_whitespace(result); llama_unescape_whitespace(result);
} else if (llama_vocab_get_type(model->vocab) == LLAMA_VOCAB_TYPE_BPE) { } else if (llama_vocab_get_type(model->vocab) == LLAMA_VOCAB_TYPE_BPE) {
result = llama_decode_text(result); result = llama_decode_text(result);
} else {
GGML_ASSERT(false);
} }
if (length < (int) result.length()) { if (length < (int) result.length()) {
return -result.length(); return -result.length();
@ -7249,14 +7255,15 @@ int llama_token_to_piece_with_model(const struct llama_model * model, llama_toke
} }
buf[0] = llama_token_to_byte(model->vocab, token); buf[0] = llama_token_to_byte(model->vocab, token);
return 1; return 1;
} } else if (llama_vocab_get_type(model->vocab) == LLAMA_VOCAB_TYPE_BPE) {
else {
std::string result = llama_decode_text(model->vocab.id_to_token[token].text); std::string result = llama_decode_text(model->vocab.id_to_token[token].text);
if (length < (int)result.length()) { if (length < (int)result.length()) {
return -result.length(); return -result.length();
} }
memcpy(buf, result.c_str(), result.length()); memcpy(buf, result.c_str(), result.length());
return result.length(); return result.length();
} else {
GGML_ASSERT(false);
} }
} }
} }