Update src/llama.cpp
Co-authored-by: compilade <git@compilade.net>
This commit is contained in:
parent
7647916c92
commit
689e38cacc
1 changed files with 1 additions and 0 deletions
|
@ -5533,6 +5533,7 @@ static void llm_load_vocab(
|
|||
} else if (
|
||||
tokenizer_pre == "smollm") {
|
||||
vocab.type_pre = LLAMA_VOCAB_PRE_TYPE_SMOLLM;
|
||||
vocab.tokenizer_clean_spaces = false;
|
||||
} else {
|
||||
throw std::runtime_error(format("unknown pre-tokenizer type: '%s'", tokenizer_pre.c_str()));
|
||||
}
|
||||
|
|
Loading…
Add table
Add a link
Reference in a new issue