vbatts/llama.cpp
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Updated/merged the deepseek coder pr

Browse source
This commit is contained in:
Jaggzh 2024-02-12 04:18:06 -08:00 • committed by Georgi Gerganov
parent 4056dc5b1e
commit c8e7d9521d
No known key found for this signature in database
GPG key ID: 449E073F9DC10735
1 changed files with 10 additions and 10 deletions
Download patch file Download diff file Expand all files Collapse all files

20
llama.cpp
View file

@ -4228,16 +4228,6 @@ static void llm_load_vocab(
if (add_space_prefix_keyidx != -1) { if (add_space_prefix_keyidx != -1) {
vocab.add_space_prefix = gguf_get_val_bool(ctx, add_space_prefix_keyidx); vocab.add_space_prefix = gguf_get_val_bool(ctx, add_space_prefix_keyidx);
} // The default value of add_space_prefix is true. } // The default value of add_space_prefix is true.
} else if (tokenizer_name == "bert") {
vocab.type = LLAMA_VOCAB_TYPE_WPM;
// default special tokens
vocab.special_bos_id = 101;
vocab.special_eos_id = 102;
vocab.special_unk_id = 100;
vocab.special_sep_id = -1;
vocab.special_pad_id = -1;
vocab.add_space_prefix = false;
} else { } else {
if (tokenizer_name == "gpt2") { if (tokenizer_name == "gpt2") {
vocab.type = LLAMA_VOCAB_TYPE_BPE; vocab.type = LLAMA_VOCAB_TYPE_BPE;
@ -4245,6 +4235,16 @@ static void llm_load_vocab(
vocab.type = LLAMA_VOCAB_TYPE_DEEPSEEKCODER; vocab.type = LLAMA_VOCAB_TYPE_DEEPSEEKCODER;
} else if (tokenizer_name == "deepseek_llm") { } else if (tokenizer_name == "deepseek_llm") {
vocab.type = LLAMA_VOCAB_TYPE_DEEPSEEKLLM; vocab.type = LLAMA_VOCAB_TYPE_DEEPSEEKLLM;
} else if (tokenizer_name == "bert") {
vocab.type = LLAMA_VOCAB_TYPE_WPM;
// default special tokens
vocab.special_bos_id = 101;
vocab.special_eos_id = 102;
vocab.special_unk_id = 100;
vocab.special_sep_id = -1;
vocab.special_pad_id = -1;
vocab.add_space_prefix = false;
} else { } else {
LLAMA_LOG_WARN("%s: unknown tokenizer: '%s'", __func__, tokenizer_name.c_str()); LLAMA_LOG_WARN("%s: unknown tokenizer: '%s'", __func__, tokenizer_name.c_str());
LLAMA_LOG_WARN("%s: using default tokenizer: 'llama'", __func__); LLAMA_LOG_WARN("%s: using default tokenizer: 'llama'", __func__);