vbatts/llama.cpp
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Merge branch 'glm_support'

Browse source
This commit is contained in:
toyer 2024-06-25 06:26:49 +00:00
commit 3557944893
1 changed files with 8 additions and 6 deletions
Download patch file Download diff file Expand all files Collapse all files

14
llama.cpp
View file

@ -4836,10 +4836,6 @@ static void llm_load_vocab(
return; return;
} else if (tokenizer_model == "llama") { } else if (tokenizer_model == "llama") {
vocab.type = LLAMA_VOCAB_TYPE_SPM; vocab.type = LLAMA_VOCAB_TYPE_SPM;
// chatglm3 needs to preprocess prefix and suffix
if (tokenizer_pre == "chatglm-spm") {
vocab.type_pre = LLAMA_VOCAB_PRE_TYPE_CHATGLM3;
}
// default special tokens // default special tokens
vocab.special_bos_id = 1; vocab.special_bos_id = 1;
@ -4988,6 +4984,13 @@ static void llm_load_vocab(
vocab.type_pre = LLAMA_VOCAB_PRE_TYPE_DEFAULT; vocab.type_pre = LLAMA_VOCAB_PRE_TYPE_DEFAULT;
vocab.tokenizer_add_bos = true; vocab.tokenizer_add_bos = true;
vocab.tokenizer_add_eos = false; vocab.tokenizer_add_eos = false;
// chatglm3 needs to preprocess prefix and suffix
if (tokenizer_pre == "chatglm-spm") {
vocab.type_pre = LLAMA_VOCAB_PRE_TYPE_CHATGLM3;
vocab.tokenizer_add_bos = false;
vocab.tokenizer_add_eos = false;
vocab.tokenizer_add_space_prefix = false;
}
} else if (vocab.type == LLAMA_VOCAB_TYPE_WPM) { } else if (vocab.type == LLAMA_VOCAB_TYPE_WPM) {
vocab.type_pre = LLAMA_VOCAB_PRE_TYPE_DEFAULT; vocab.type_pre = LLAMA_VOCAB_PRE_TYPE_DEFAULT;
vocab.tokenizer_add_bos = true; vocab.tokenizer_add_bos = true;
@ -5085,7 +5088,7 @@ static void llm_load_vocab(
vocab.special_eot_id = 107; vocab.special_eot_id = 107;
} }
} }
try { try {
vocab.linefeed_id = llama_byte_to_token(vocab, '\n'); vocab.linefeed_id = llama_byte_to_token(vocab, '\n');
} catch (const std::exception & e) { } catch (const std::exception & e) {
@ -14220,7 +14223,6 @@ static std::vector<llama_vocab::id> llama_tokenize_internal(const llama_vocab &
// tokenizer.encode('', add_special_tokens=False) returns [] // tokenizer.encode('', add_special_tokens=False) returns []
bool is_prev_special = false; bool is_prev_special = false;
if (add_special && vocab.tokenizer_add_bos) { if (add_special && vocab.tokenizer_add_bos) {
GGML_ASSERT(vocab.special_bos_id != -1); GGML_ASSERT(vocab.special_bos_id != -1);
output.push_back(vocab.special_bos_id); output.push_back(vocab.special_bos_id);