vbatts/llama.cpp
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

llama : use new pre-tokenizer type

Browse source
This commit is contained in:
Georgi Gerganov 2024-04-26 20:08:28 +03:00
parent 9b4d63ae53
commit 43e12ce8e5
No known key found for this signature in database
GPG key ID: 449E073F9DC10735
12 changed files with 87 additions and 44 deletions
Download patch file Download diff file Expand all files Collapse all files

8
llama.h
View file

@ -69,6 +69,14 @@ extern "C" {
LLAMA_VOCAB_TYPE_WPM = 3, // BERT tokenizer based on WordPiece
};
// pre-tokenization types
enum llama_vocab_pre_type {
LLAMA_VOCAB_PRE_TYPE_DEFAULT = 0,
LLAMA_VOCAB_PRE_TYPE_LLAMA3 = 1,
LLAMA_VOCAB_PRE_TYPE_DEEPSEEK_LLM = 2,
LLAMA_VOCAB_PRE_TYPE_DEEPSEEK_CODER = 3,
};
// note: these values should be synchronized with ggml_rope
// TODO: maybe move this enum to ggml.h (ggml_rope_type)
enum llama_rope_type {