From 9576df906a0bb089084f77113b8c8a5fb3b37989 Mon Sep 17 00:00:00 2001 From: ngxson Date: Wed, 21 Feb 2024 11:11:29 +0100 Subject: [PATCH] server: fallback to chatml --- examples/server/server.cpp | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/examples/server/server.cpp b/examples/server/server.cpp index eb01729fa..05c2707ec 100644 --- a/examples/server/server.cpp +++ b/examples/server/server.cpp @@ -400,6 +400,16 @@ struct llama_server_context return true; } + void validate_model_chat_template(server_params & sparams) { + llama_chat_message chat[] = {{"user", "test"}}; + std::vector buf(1); + int res = llama_chat_apply_template(model, nullptr, chat, 1, true, buf.data(), buf.size()); + if (res < 0) { + LOG_ERROR("The chat template comes with this model is not yet supported, falling back to chatml. This may cause the model to output suboptimal responses", {}); + sparams.chat_template = "<|im_start|>"; // llama_chat_apply_template only checks if <|im_start|> exist in the template + } + } + void initialize() { // create slots all_slots_are_idle = true; @@ -2713,6 +2723,9 @@ int main(int argc, char **argv) LOG_INFO("model loaded", {}); } + // check if the template comes with the model is supported by us + llama.validate_model_chat_template(sparams); + // Middleware for API key validation auto validate_api_key = [&sparams](const httplib::Request &req, httplib::Response &res) -> bool { // If API key is not set, skip validation