vbatts/llama.cpp
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Updated/merged the deepseek coder pr

Browse source
This commit is contained in:
Jaggzh 2024-02-12 04:18:06 -08:00 • committed by Georgi Gerganov
parent 4056dc5b1e
commit c8e7d9521d
No known key found for this signature in database
GPG key ID: 449E073F9DC10735
1 changed files with 10 additions and 10 deletions
Download patch file Download diff file Expand all files Collapse all files

14
llama.cpp
View file

@ -4228,6 +4228,13 @@ static void llm_load_vocab(
if (add_space_prefix_keyidx != -1) {
vocab.add_space_prefix = gguf_get_val_bool(ctx, add_space_prefix_keyidx);
} // The default value of add_space_prefix is true.
} else {
if (tokenizer_name == "gpt2") {
vocab.type = LLAMA_VOCAB_TYPE_BPE;
} else if (tokenizer_name == "deepseek_coder") {
vocab.type = LLAMA_VOCAB_TYPE_DEEPSEEKCODER;
} else if (tokenizer_name == "deepseek_llm") {
vocab.type = LLAMA_VOCAB_TYPE_DEEPSEEKLLM;
} else if (tokenizer_name == "bert") {
vocab.type = LLAMA_VOCAB_TYPE_WPM;
@ -4238,13 +4245,6 @@ static void llm_load_vocab(
vocab.special_sep_id = -1;
vocab.special_pad_id = -1;
vocab.add_space_prefix = false;
} else {
if (tokenizer_name == "gpt2") {
vocab.type = LLAMA_VOCAB_TYPE_BPE;
} else if (tokenizer_name == "deepseek_coder") {
vocab.type = LLAMA_VOCAB_TYPE_DEEPSEEKCODER;
} else if (tokenizer_name == "deepseek_llm") {
vocab.type = LLAMA_VOCAB_TYPE_DEEPSEEKLLM;
} else {
LLAMA_LOG_WARN("%s: unknown tokenizer: '%s'", __func__, tokenizer_name.c_str());
LLAMA_LOG_WARN("%s: using default tokenizer: 'llama'", __func__);