vbatts/llama.cpp
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Update src/llama.cpp

Browse source 
Co-authored-by: compilade <git@compilade.net>
This commit is contained in:
Jason Stillerman 2024-07-21 02:49:10 -04:00 • committed by GitHub
parent 7647916c92
commit 689e38cacc
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
1 changed files with 1 additions and 0 deletions
Download patch file Download diff file Expand all files Collapse all files

1
src/llama.cpp
View file

@ -5533,6 +5533,7 @@ static void llm_load_vocab(
} else if ( } else if (
tokenizer_pre == "smollm") { tokenizer_pre == "smollm") {
vocab.type_pre = LLAMA_VOCAB_PRE_TYPE_SMOLLM; vocab.type_pre = LLAMA_VOCAB_PRE_TYPE_SMOLLM;
vocab.tokenizer_clean_spaces = false;
} else { } else {
throw std::runtime_error(format("unknown pre-tokenizer type: '%s'", tokenizer_pre.c_str())); throw std::runtime_error(format("unknown pre-tokenizer type: '%s'", tokenizer_pre.c_str()));
} }