llama : match EOT token <|end|>

This commit is contained in:
Georgi Gerganov 2024-04-24 09:39:04 +03:00
parent cef12f9e45
commit 1bf93ced81
No known key found for this signature in database
GPG key ID: 449E073F9DC10735

View file

@ -4381,6 +4381,7 @@ static void llm_load_vocab(
//vocab.id_to_token[t.second].type == LLAMA_TOKEN_TYPE_CONTROL &&
(t.first == "<|eot_id|>" ||
t.first == "<|im_end|>" ||
t.first == "<|end|>" ||
t.first == "<end_of_turn>"
)
) {