From 588b72d9500a0c29dc5ad73f74e7f2bb24d0b4fe Mon Sep 17 00:00:00 2001 From: ngxson Date: Wed, 24 Apr 2024 15:20:21 +0200 Subject: [PATCH] fix templates not support system message --- llama.cpp | 24 +++++++++++++++--------- tests/test-chat-template.cpp | 8 ++++---- 2 files changed, 19 insertions(+), 13 deletions(-) diff --git a/llama.cpp b/llama.cpp index 436082aca..76e0367b8 100644 --- a/llama.cpp +++ b/llama.cpp @@ -17074,7 +17074,7 @@ static std::string trim(const std::string & str) { return str.substr(start, end - start); } -static int32_t llama_chat_get_model_template( +LLAMA_API int32_t llama_chat_get_model_template( const struct llama_model * model, const char * name, char * buf, @@ -17110,7 +17110,7 @@ static int32_t llama_chat_get_model_template( } } -static llama_chat_template llama_chat_get_template_type(const char * tmpl) { +LLAMA_API llama_chat_template llama_chat_get_template_type(const char * tmpl) { if (tmpl == nullptr) { return LLAMA_CHAT_TEMPLATE_NOT_SUPPORTED; } @@ -17161,7 +17161,7 @@ static llama_chat_template llama_chat_get_template_type(const char * tmpl) { } } -static int32_t llama_chat_get_prefix( +LLAMA_API int32_t llama_chat_get_prefix( const llama_chat_template tmpl, const char * role, const char * prev_role, @@ -17266,7 +17266,7 @@ static int32_t llama_chat_get_prefix( return output.size(); } -static int32_t llama_chat_get_postfix( +LLAMA_API int32_t llama_chat_get_postfix( const llama_chat_template tmpl, const char * role, const char * prev_role, @@ -17345,7 +17345,7 @@ static int32_t llama_chat_get_postfix( return output.size(); } -static bool llama_chat_support_system_message(const llama_chat_template tmpl) { +LLAMA_API bool llama_chat_support_system_message(const llama_chat_template tmpl) { switch (tmpl) { case LLAMA_CHAT_TEMPLATE_CHATML: case LLAMA_CHAT_TEMPLATE_LLAMA2_SYS_BOS: @@ -17377,7 +17377,7 @@ LLAMA_API int32_t llama_chat_apply_template( int32_t res = llama_chat_get_model_template(model, nullptr, model_template.data(), model_template.size()); if (res < 0) { // worst case: there is no information about template, we will use chatml by default - curr_tmpl = "chatml"; // see llama_chat_apply_template_internal + curr_tmpl = "chatml"; } else { curr_tmpl = std::string(model_template.data(), model_template.size()); } @@ -17393,12 +17393,18 @@ LLAMA_API int32_t llama_chat_apply_template( // format the chat to string std::stringstream ss; std::string prev_role; + bool merge_system_message = false; for (size_t i = 0; i < n_msg; i++) { std::string role(chat[i].role); std::string content(chat[i].content); - if (!support_system_message) { - // if the template does not support system message, we convert it to user message - role = role == "system" ? "user" : role; + // if the template does not support system message, we merge it with the next message + if (role == "system" && !support_system_message) { + merge_system_message = true; + continue; + } + if (merge_system_message && i > 0) { + content = std::string(chat[i - 1].content) + "\n\n" + content; + merge_system_message = false; } std::vector prefix(1024, 0); std::vector postfix(1024, 0); diff --git a/tests/test-chat-template.cpp b/tests/test-chat-template.cpp index c2db07257..ebdb84bd5 100644 --- a/tests/test-chat-template.cpp +++ b/tests/test-chat-template.cpp @@ -54,7 +54,7 @@ int main(void) { // teknium/OpenHermes-2.5-Mistral-7B "<|im_start|>system\nYou are a helpful assistant<|im_end|>\n<|im_start|>user\nHello<|im_end|>\n<|im_start|>assistant\nHi there<|im_end|>\n<|im_start|>user\nWho are you<|im_end|>\n<|im_start|>assistant\nI am an assistant<|im_end|>\n<|im_start|>user\nAnother question<|im_end|>\n<|im_start|>assistant\n", // mistralai/Mistral-7B-Instruct-v0.2 - "[INST] You are a helpful assistant [/INST][INST] Hello [/INST]Hi there[INST] Who are you [/INST]I am an assistant[INST] Another question [/INST]", + "[INST] You are a helpful assistant\n\nHello [/INST]Hi there[INST] Who are you [/INST]I am an assistant[INST] Another question [/INST]", // TheBloke/FusionNet_34Bx2_MoE-AWQ "[INST] <>\nYou are a helpful assistant\n<>\n\nHello [/INST] Hi there [INST] Who are you [/INST] I am an assistant [INST] Another question [/INST]", // bofenghuang/vigogne-2-70b-chat @@ -62,15 +62,15 @@ int main(void) { // mlabonne/AlphaMonarch-7B "system\nYou are a helpful assistant\nuser\nHello\nassistant\nHi there\nuser\nWho are you\nassistant\nI am an assistant\nuser\nAnother question\nassistant\n", // google/gemma-7b-it - "user\nYou are a helpful assistant\nuser\nHello\nmodel\nHi there\nuser\nWho are you\nmodel\nI am an assistant\nuser\nAnother question\nmodel\n", + "user\nYou are a helpful assistant\n\nHello\nmodel\nHi there\nuser\nWho are you\nmodel\nI am an assistant\nuser\nAnother question\nmodel\n", // OrionStarAI/Orion-14B-Chat "System: You are a helpful assistant\n\nHuman: Hello\n\nAssistant: Hi thereHuman: Who are you\n\nAssistant: I am an assistantHuman: Another question\n\nAssistant: ", // openchat/openchat-3.5-0106 "You are a helpful assistant<|end_of_turn|>GPT4 Correct User: Hello<|end_of_turn|>GPT4 Correct Assistant: Hi there<|end_of_turn|>GPT4 Correct User: Who are you<|end_of_turn|>GPT4 Correct Assistant: I am an assistant<|end_of_turn|>GPT4 Correct User: Another question<|end_of_turn|>GPT4 Correct Assistant:", // deepseek-ai/deepseek-coder-33b-instruct - "### Instruction:\nYou are a helpful assistant\n### Instruction:\nHello\n### Response:\nHi there\n<|EOT|>\n### Instruction:\nWho are you\n### Response:\nI am an assistant\n<|EOT|>\n### Instruction:\nAnother question\n### Response:\n", + "### Instruction:\nYou are a helpful assistant\n\nHello\n### Response:\nHi there\n<|EOT|>\n### Instruction:\nWho are you\n### Response:\nI am an assistant\n<|EOT|>\n### Instruction:\nAnother question\n### Response:\n", // eachadea/vicuna-13b-1.1 - "USER: You are a helpful assistant\nUSER: Hello\nASSISTANT: Hi there\nUSER: Who are you\nASSISTANT: I am an assistant\nUSER: Another question\nASSISTANT:", + "USER: You are a helpful assistant\n\nHello\nASSISTANT: Hi there\nUSER: Who are you\nASSISTANT: I am an assistant\nUSER: Another question\nASSISTANT:", // Orca-Vicuna "SYSTEM: You are a helpful assistant\nUSER: Hello\nASSISTANT: Hi there\nUSER: Who are you\nASSISTANT: I am an assistant\nUSER: Another question\nASSISTANT:", // CohereForAI/c4ai-command-r-plus