llama : match EOT token <|end|>
This commit is contained in:
parent
cef12f9e45
commit
1bf93ced81
1 changed files with 1 additions and 0 deletions
|
@ -4381,6 +4381,7 @@ static void llm_load_vocab(
|
|||
//vocab.id_to_token[t.second].type == LLAMA_TOKEN_TYPE_CONTROL &&
|
||||
(t.first == "<|eot_id|>" ||
|
||||
t.first == "<|im_end|>" ||
|
||||
t.first == "<|end|>" ||
|
||||
t.first == "<end_of_turn>"
|
||||
)
|
||||
) {
|
||||
|
|
Loading…
Add table
Add a link
Reference in a new issue