vbatts/llama.cpp
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

fix templates not support system message

Browse source
This commit is contained in:
ngxson 2024-04-24 15:20:21 +02:00
parent a202b56127
commit 588b72d950
2 changed files with 19 additions and 13 deletions
Download patch file Download diff file Expand all files Collapse all files

24
llama.cpp
View file

@ -17074,7 +17074,7 @@ static std::string trim(const std::string & str) {
return str.substr(start, end - start); return str.substr(start, end - start);
} }
static int32_t llama_chat_get_model_template( LLAMA_API int32_t llama_chat_get_model_template(
const struct llama_model * model, const struct llama_model * model,
const char * name, const char * name,
char * buf, char * buf,
@ -17110,7 +17110,7 @@ static int32_t llama_chat_get_model_template(
} }
} }
static llama_chat_template llama_chat_get_template_type(const char * tmpl) { LLAMA_API llama_chat_template llama_chat_get_template_type(const char * tmpl) {
if (tmpl == nullptr) { if (tmpl == nullptr) {
return LLAMA_CHAT_TEMPLATE_NOT_SUPPORTED; return LLAMA_CHAT_TEMPLATE_NOT_SUPPORTED;
} }
@ -17161,7 +17161,7 @@ static llama_chat_template llama_chat_get_template_type(const char * tmpl) {
} }
} }
static int32_t llama_chat_get_prefix( LLAMA_API int32_t llama_chat_get_prefix(
const llama_chat_template tmpl, const llama_chat_template tmpl,
const char * role, const char * role,
const char * prev_role, const char * prev_role,
@ -17266,7 +17266,7 @@ static int32_t llama_chat_get_prefix(
return output.size(); return output.size();
} }
static int32_t llama_chat_get_postfix( LLAMA_API int32_t llama_chat_get_postfix(
const llama_chat_template tmpl, const llama_chat_template tmpl,
const char * role, const char * role,
const char * prev_role, const char * prev_role,
@ -17345,7 +17345,7 @@ static int32_t llama_chat_get_postfix(
return output.size(); return output.size();
} }
static bool llama_chat_support_system_message(const llama_chat_template tmpl) { LLAMA_API bool llama_chat_support_system_message(const llama_chat_template tmpl) {
switch (tmpl) { switch (tmpl) {
case LLAMA_CHAT_TEMPLATE_CHATML: case LLAMA_CHAT_TEMPLATE_CHATML:
case LLAMA_CHAT_TEMPLATE_LLAMA2_SYS_BOS: case LLAMA_CHAT_TEMPLATE_LLAMA2_SYS_BOS:
@ -17377,7 +17377,7 @@ LLAMA_API int32_t llama_chat_apply_template(
int32_t res = llama_chat_get_model_template(model, nullptr, model_template.data(), model_template.size()); int32_t res = llama_chat_get_model_template(model, nullptr, model_template.data(), model_template.size());
if (res < 0) { if (res < 0) {
// worst case: there is no information about template, we will use chatml by default // worst case: there is no information about template, we will use chatml by default
curr_tmpl = "chatml"; // see llama_chat_apply_template_internal curr_tmpl = "chatml";
} else { } else {
curr_tmpl = std::string(model_template.data(), model_template.size()); curr_tmpl = std::string(model_template.data(), model_template.size());
} }
@ -17393,12 +17393,18 @@ LLAMA_API int32_t llama_chat_apply_template(
// format the chat to string // format the chat to string
std::stringstream ss; std::stringstream ss;
std::string prev_role; std::string prev_role;
bool merge_system_message = false;
for (size_t i = 0; i < n_msg; i++) { for (size_t i = 0; i < n_msg; i++) {
std::string role(chat[i].role); std::string role(chat[i].role);
std::string content(chat[i].content); std::string content(chat[i].content);
if (!support_system_message) { // if the template does not support system message, we merge it with the next message
// if the template does not support system message, we convert it to user message if (role == "system" && !support_system_message) {
role = role == "system" ? "user" : role; merge_system_message = true;
continue;
}
if (merge_system_message && i > 0) {
content = std::string(chat[i - 1].content) + "\n\n" + content;
merge_system_message = false;
} }
std::vector<char> prefix(1024, 0); std::vector<char> prefix(1024, 0);
std::vector<char> postfix(1024, 0); std::vector<char> postfix(1024, 0);

8
tests/test-chat-template.cpp
View file

@ -54,7 +54,7 @@ int main(void) {
// teknium/OpenHermes-2.5-Mistral-7B // teknium/OpenHermes-2.5-Mistral-7B
"<|im_start|>system\nYou are a helpful assistant<|im_end|>\n<|im_start|>user\nHello<|im_end|>\n<|im_start|>assistant\nHi there<|im_end|>\n<|im_start|>user\nWho are you<|im_end|>\n<|im_start|>assistant\nI am an assistant<|im_end|>\n<|im_start|>user\nAnother question<|im_end|>\n<|im_start|>assistant\n", "<|im_start|>system\nYou are a helpful assistant<|im_end|>\n<|im_start|>user\nHello<|im_end|>\n<|im_start|>assistant\nHi there<|im_end|>\n<|im_start|>user\nWho are you<|im_end|>\n<|im_start|>assistant\nI am an assistant<|im_end|>\n<|im_start|>user\nAnother question<|im_end|>\n<|im_start|>assistant\n",
// mistralai/Mistral-7B-Instruct-v0.2 // mistralai/Mistral-7B-Instruct-v0.2
"[INST] You are a helpful assistant [/INST][INST] Hello [/INST]Hi there</s>[INST] Who are you [/INST]I am an assistant</s>[INST] Another question [/INST]", "[INST] You are a helpful assistant\n\nHello [/INST]Hi there</s>[INST] Who are you [/INST]I am an assistant</s>[INST] Another question [/INST]",
// TheBloke/FusionNet_34Bx2_MoE-AWQ // TheBloke/FusionNet_34Bx2_MoE-AWQ
"[INST] <<SYS>>\nYou are a helpful assistant\n<</SYS>>\n\nHello [/INST] Hi there </s><s>[INST] Who are you [/INST] I am an assistant </s><s>[INST] Another question [/INST]", "[INST] <<SYS>>\nYou are a helpful assistant\n<</SYS>>\n\nHello [/INST] Hi there </s><s>[INST] Who are you [/INST] I am an assistant </s><s>[INST] Another question [/INST]",
// bofenghuang/vigogne-2-70b-chat // bofenghuang/vigogne-2-70b-chat
@ -62,15 +62,15 @@ int main(void) {
// mlabonne/AlphaMonarch-7B // mlabonne/AlphaMonarch-7B
"system\nYou are a helpful assistant</s>\n<s>user\nHello</s>\n<s>assistant\nHi there</s>\n<s>user\nWho are you</s>\n<s>assistant\nI am an assistant</s>\n<s>user\nAnother question</s>\n<s>assistant\n", "system\nYou are a helpful assistant</s>\n<s>user\nHello</s>\n<s>assistant\nHi there</s>\n<s>user\nWho are you</s>\n<s>assistant\nI am an assistant</s>\n<s>user\nAnother question</s>\n<s>assistant\n",
// google/gemma-7b-it // google/gemma-7b-it
"<start_of_turn>user\nYou are a helpful assistant<end_of_turn>\n<start_of_turn>user\nHello<end_of_turn>\n<start_of_turn>model\nHi there<end_of_turn>\n<start_of_turn>user\nWho are you<end_of_turn>\n<start_of_turn>model\nI am an assistant<end_of_turn>\n<start_of_turn>user\nAnother question<end_of_turn>\n<start_of_turn>model\n", "<start_of_turn>user\nYou are a helpful assistant\n\nHello<end_of_turn>\n<start_of_turn>model\nHi there<end_of_turn>\n<start_of_turn>user\nWho are you<end_of_turn>\n<start_of_turn>model\nI am an assistant<end_of_turn>\n<start_of_turn>user\nAnother question<end_of_turn>\n<start_of_turn>model\n",
// OrionStarAI/Orion-14B-Chat // OrionStarAI/Orion-14B-Chat
"System: You are a helpful assistant\n\nHuman: Hello\n\nAssistant: </s>Hi there</s>Human: Who are you\n\nAssistant: </s>I am an assistant</s>Human: Another question\n\nAssistant: </s>", "System: You are a helpful assistant\n\nHuman: Hello\n\nAssistant: </s>Hi there</s>Human: Who are you\n\nAssistant: </s>I am an assistant</s>Human: Another question\n\nAssistant: </s>",
// openchat/openchat-3.5-0106 // openchat/openchat-3.5-0106
"You are a helpful assistant<|end_of_turn|>GPT4 Correct User: Hello<|end_of_turn|>GPT4 Correct Assistant: Hi there<|end_of_turn|>GPT4 Correct User: Who are you<|end_of_turn|>GPT4 Correct Assistant: I am an assistant<|end_of_turn|>GPT4 Correct User: Another question<|end_of_turn|>GPT4 Correct Assistant:", "You are a helpful assistant<|end_of_turn|>GPT4 Correct User: Hello<|end_of_turn|>GPT4 Correct Assistant: Hi there<|end_of_turn|>GPT4 Correct User: Who are you<|end_of_turn|>GPT4 Correct Assistant: I am an assistant<|end_of_turn|>GPT4 Correct User: Another question<|end_of_turn|>GPT4 Correct Assistant:",
// deepseek-ai/deepseek-coder-33b-instruct // deepseek-ai/deepseek-coder-33b-instruct
"### Instruction:\nYou are a helpful assistant\n### Instruction:\nHello\n### Response:\nHi there\n<|EOT|>\n### Instruction:\nWho are you\n### Response:\nI am an assistant\n<|EOT|>\n### Instruction:\nAnother question\n### Response:\n", "### Instruction:\nYou are a helpful assistant\n\nHello\n### Response:\nHi there\n<|EOT|>\n### Instruction:\nWho are you\n### Response:\nI am an assistant\n<|EOT|>\n### Instruction:\nAnother question\n### Response:\n",
// eachadea/vicuna-13b-1.1 // eachadea/vicuna-13b-1.1
"USER: You are a helpful assistant\nUSER: Hello\nASSISTANT: Hi there</s>\nUSER: Who are you\nASSISTANT: I am an assistant</s>\nUSER: Another question\nASSISTANT:", "USER: You are a helpful assistant\n\nHello\nASSISTANT: Hi there</s>\nUSER: Who are you\nASSISTANT: I am an assistant</s>\nUSER: Another question\nASSISTANT:",
// Orca-Vicuna // Orca-Vicuna
"SYSTEM: You are a helpful assistant\nUSER: Hello\nASSISTANT: Hi there</s>\nUSER: Who are you\nASSISTANT: I am an assistant</s>\nUSER: Another question\nASSISTANT:", "SYSTEM: You are a helpful assistant\nUSER: Hello\nASSISTANT: Hi there</s>\nUSER: Who are you\nASSISTANT: I am an assistant</s>\nUSER: Another question\nASSISTANT:",
// CohereForAI/c4ai-command-r-plus // CohereForAI/c4ai-command-r-plus