vbatts/llama.cpp
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

fix templates not support system message

Browse source
This commit is contained in:
ngxson 2024-04-24 15:20:21 +02:00
parent a202b56127
commit 588b72d950
2 changed files with 19 additions and 13 deletions
Download patch file Download diff file Expand all files Collapse all files

24
llama.cpp
View file

@ -17074,7 +17074,7 @@ static std::string trim(const std::string & str) {
return str.substr(start, end - start);
}
static int32_t llama_chat_get_model_template(
LLAMA_API int32_t llama_chat_get_model_template(
const struct llama_model * model,
const char * name,
char * buf,
@ -17110,7 +17110,7 @@ static int32_t llama_chat_get_model_template(
}
}
static llama_chat_template llama_chat_get_template_type(const char * tmpl) {
LLAMA_API llama_chat_template llama_chat_get_template_type(const char * tmpl) {
if (tmpl == nullptr) {
return LLAMA_CHAT_TEMPLATE_NOT_SUPPORTED;
}
@ -17161,7 +17161,7 @@ static llama_chat_template llama_chat_get_template_type(const char * tmpl) {
}
}
static int32_t llama_chat_get_prefix(
LLAMA_API int32_t llama_chat_get_prefix(
const llama_chat_template tmpl,
const char * role,
const char * prev_role,
@ -17266,7 +17266,7 @@ static int32_t llama_chat_get_prefix(
return output.size();
}
static int32_t llama_chat_get_postfix(
LLAMA_API int32_t llama_chat_get_postfix(
const llama_chat_template tmpl,
const char * role,
const char * prev_role,
@ -17345,7 +17345,7 @@ static int32_t llama_chat_get_postfix(
return output.size();
}
static bool llama_chat_support_system_message(const llama_chat_template tmpl) {
LLAMA_API bool llama_chat_support_system_message(const llama_chat_template tmpl) {
switch (tmpl) {
case LLAMA_CHAT_TEMPLATE_CHATML:
case LLAMA_CHAT_TEMPLATE_LLAMA2_SYS_BOS:
@ -17377,7 +17377,7 @@ LLAMA_API int32_t llama_chat_apply_template(
int32_t res = llama_chat_get_model_template(model, nullptr, model_template.data(), model_template.size());
if (res < 0) {
// worst case: there is no information about template, we will use chatml by default
curr_tmpl = "chatml"; // see llama_chat_apply_template_internal
curr_tmpl = "chatml";
} else {
curr_tmpl = std::string(model_template.data(), model_template.size());
}
@ -17393,12 +17393,18 @@ LLAMA_API int32_t llama_chat_apply_template(
// format the chat to string
std::stringstream ss;
std::string prev_role;
bool merge_system_message = false;
for (size_t i = 0; i < n_msg; i++) {
std::string role(chat[i].role);
std::string content(chat[i].content);
if (!support_system_message) {
// if the template does not support system message, we convert it to user message
role = role == "system" ? "user" : role;
// if the template does not support system message, we merge it with the next message
if (role == "system" && !support_system_message) {
merge_system_message = true;
continue;
}
if (merge_system_message && i > 0) {
content = std::string(chat[i - 1].content) + "\n\n" + content;
merge_system_message = false;
}
std::vector<char> prefix(1024, 0);
std::vector<char> postfix(1024, 0);

8
tests/test-chat-template.cpp
View file

@ -54,7 +54,7 @@ int main(void) {
// teknium/OpenHermes-2.5-Mistral-7B
"<|im_start|>system\nYou are a helpful assistant<|im_end|>\n<|im_start|>user\nHello<|im_end|>\n<|im_start|>assistant\nHi there<|im_end|>\n<|im_start|>user\nWho are you<|im_end|>\n<|im_start|>assistant\nI am an assistant<|im_end|>\n<|im_start|>user\nAnother question<|im_end|>\n<|im_start|>assistant\n",
// mistralai/Mistral-7B-Instruct-v0.2
"[INST] You are a helpful assistant [/INST][INST] Hello [/INST]Hi there</s>[INST] Who are you [/INST]I am an assistant</s>[INST] Another question [/INST]",
"[INST] You are a helpful assistant\n\nHello [/INST]Hi there</s>[INST] Who are you [/INST]I am an assistant</s>[INST] Another question [/INST]",
// TheBloke/FusionNet_34Bx2_MoE-AWQ
"[INST] <<SYS>>\nYou are a helpful assistant\n<</SYS>>\n\nHello [/INST] Hi there </s><s>[INST] Who are you [/INST] I am an assistant </s><s>[INST] Another question [/INST]",
// bofenghuang/vigogne-2-70b-chat
@ -62,15 +62,15 @@ int main(void) {
// mlabonne/AlphaMonarch-7B
"system\nYou are a helpful assistant</s>\n<s>user\nHello</s>\n<s>assistant\nHi there</s>\n<s>user\nWho are you</s>\n<s>assistant\nI am an assistant</s>\n<s>user\nAnother question</s>\n<s>assistant\n",
// google/gemma-7b-it
"<start_of_turn>user\nYou are a helpful assistant<end_of_turn>\n<start_of_turn>user\nHello<end_of_turn>\n<start_of_turn>model\nHi there<end_of_turn>\n<start_of_turn>user\nWho are you<end_of_turn>\n<start_of_turn>model\nI am an assistant<end_of_turn>\n<start_of_turn>user\nAnother question<end_of_turn>\n<start_of_turn>model\n",
"<start_of_turn>user\nYou are a helpful assistant\n\nHello<end_of_turn>\n<start_of_turn>model\nHi there<end_of_turn>\n<start_of_turn>user\nWho are you<end_of_turn>\n<start_of_turn>model\nI am an assistant<end_of_turn>\n<start_of_turn>user\nAnother question<end_of_turn>\n<start_of_turn>model\n",
// OrionStarAI/Orion-14B-Chat
"System: You are a helpful assistant\n\nHuman: Hello\n\nAssistant: </s>Hi there</s>Human: Who are you\n\nAssistant: </s>I am an assistant</s>Human: Another question\n\nAssistant: </s>",
// openchat/openchat-3.5-0106
"You are a helpful assistant<|end_of_turn|>GPT4 Correct User: Hello<|end_of_turn|>GPT4 Correct Assistant: Hi there<|end_of_turn|>GPT4 Correct User: Who are you<|end_of_turn|>GPT4 Correct Assistant: I am an assistant<|end_of_turn|>GPT4 Correct User: Another question<|end_of_turn|>GPT4 Correct Assistant:",
// deepseek-ai/deepseek-coder-33b-instruct
"### Instruction:\nYou are a helpful assistant\n### Instruction:\nHello\n### Response:\nHi there\n<|EOT|>\n### Instruction:\nWho are you\n### Response:\nI am an assistant\n<|EOT|>\n### Instruction:\nAnother question\n### Response:\n",
"### Instruction:\nYou are a helpful assistant\n\nHello\n### Response:\nHi there\n<|EOT|>\n### Instruction:\nWho are you\n### Response:\nI am an assistant\n<|EOT|>\n### Instruction:\nAnother question\n### Response:\n",
// eachadea/vicuna-13b-1.1
"USER: You are a helpful assistant\nUSER: Hello\nASSISTANT: Hi there</s>\nUSER: Who are you\nASSISTANT: I am an assistant</s>\nUSER: Another question\nASSISTANT:",
"USER: You are a helpful assistant\n\nHello\nASSISTANT: Hi there</s>\nUSER: Who are you\nASSISTANT: I am an assistant</s>\nUSER: Another question\nASSISTANT:",
// Orca-Vicuna
"SYSTEM: You are a helpful assistant\nUSER: Hello\nASSISTANT: Hi there</s>\nUSER: Who are you\nASSISTANT: I am an assistant</s>\nUSER: Another question\nASSISTANT:",
// CohereForAI/c4ai-command-r-plus