vbatts/llama.cpp
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

llama : handle no-vocab detokenization

Browse source
This commit is contained in:
Georgi Gerganov 2024-12-16 21:45:25 +02:00
parent 824fa750d4
commit d291c74253
No known key found for this signature in database
GPG key ID: 449E073F9DC10735
1 changed files with 4 additions and 0 deletions
Download patch file Download diff file Expand all files Collapse all files

4
src/llama-vocab.cpp
View file

@ -1867,6 +1867,10 @@ int32_t llama_detokenize_impl(
int32_t text_len_max, int32_t text_len_max,
bool remove_special, bool remove_special,
bool unparse_special) { bool unparse_special) {
if (vocab.type == LLAMA_VOCAB_TYPE_NONE) {
return 0;
}
GGML_ASSERT(vocab.tokenizer && "Tokenizer not initialized. Call llama_vocab::init_tokenizer() first."); GGML_ASSERT(vocab.tokenizer && "Tokenizer not initialized. Call llama_vocab::init_tokenizer() first.");
int32_t avail = text_len_max; int32_t avail = text_len_max;