diff --git a/examples/server/tests/features/tool_call.feature b/examples/server/tests/features/tool_call.feature index cc8ba02c6..0e753fd69 100644 --- a/examples/server/tests/features/tool_call.feature +++ b/examples/server/tests/features/tool_call.feature @@ -28,17 +28,21 @@ Feature: llama.cpp server Then tool is called with arguments Examples: Prompts - | template_name | n_predict | tool_name | tool_arguments | tools | parallel_tool_calls | - | meetkai-functionary-medium-v3.1 | 128 | test | {} | [{"type":"function", "function": {"name": "test", "description": "", "parameters": {"type": "object", "properties": {}}}}] | disabled | - | meetkai-functionary-medium-v3.1 | 128 | ipython | {"code": "Yes, you can."} | [{"type":"function", "function": {"name": "ipython", "description": "", "parameters": {"type": "object", "properties": {"code": {"type": "string", "description": ""}}, "required": ["code"]}}}] | disabled | - | meetkai-functionary-medium-v3.2 | 128 | test | {} | [{"type":"function", "function": {"name": "test", "description": "", "parameters": {"type": "object", "properties": {}}}}] | disabled | - | meetkai-functionary-medium-v3.2 | 128 | ipython | {"code": "Yes,"} | [{"type":"function", "function": {"name": "ipython", "description": "", "parameters": {"type": "object", "properties": {"code": {"type": "string", "description": ""}}, "required": ["code"]}}}] | disabled | - | meta-llama-Meta-Llama-3.1-8B-Instruct | 64 | test | {} | [{"type":"function", "function": {"name": "test", "description": "", "parameters": {"type": "object", "properties": {}}}}] | disabled | - | meta-llama-Meta-Llama-3.1-8B-Instruct | 64 | ipython | {"code": "it and realed at the otter. Asked Dave Dasty, Daisy is a big, shiny blue. As"} | [{"type":"function", "function": {"name": "ipython", "description": "", "parameters": {"type": "object", "properties": {"code": {"type": "string", "description": ""}}, "required": ["code"]}}}] | disabled | - | meta-llama-Llama-3.2-3B-Instruct | 64 | test | {} | [{"type":"function", "function": {"name": "test", "description": "", "parameters": {"type": "object", "properties": {}}}}] | disabled | - | meta-llama-Llama-3.2-3B-Instruct | 64 | ipython | {"code": "Yes,"} | [{"type":"function", "function": {"name": "ipython", "description": "", "parameters": {"type": "object", "properties": {"code": {"type": "string", "description": ""}}, "required": ["code"]}}}] | disabled | - | mistralai-Mistral-Nemo-Instruct-2407 | 128 | test | {} | [{"type":"function", "function": {"name": "test", "description": "", "parameters": {"type": "object", "properties": {}}}}] | disabled | - | mistralai-Mistral-Nemo-Instruct-2407 | 128 | ipython | {"code": "It's a small cable."} | [{"type":"function", "function": {"name": "ipython", "description": "", "parameters": {"type": "object", "properties": {"code": {"type": "string", "description": ""}}, "required": ["code"]}}}] | disabled | + | template_name | n_predict | tool_name | tool_arguments | tools | parallel_tool_calls | + | meetkai-functionary-medium-v3.1 | 128 | test | {} | [{"type":"function", "function": {"name": "test", "description": "", "parameters": {"type": "object", "properties": {}}}}] | disabled | + | meetkai-functionary-medium-v3.1 | 128 | ipython | {"code": "Yes, you can."} | [{"type":"function", "function": {"name": "ipython", "description": "", "parameters": {"type": "object", "properties": {"code": {"type": "string", "description": ""}}, "required": ["code"]}}}] | disabled | + | meetkai-functionary-medium-v3.2 | 128 | test | {} | [{"type":"function", "function": {"name": "test", "description": "", "parameters": {"type": "object", "properties": {}}}}] | disabled | + | meetkai-functionary-medium-v3.2 | 128 | ipython | {"code": "Yes,"} | [{"type":"function", "function": {"name": "ipython", "description": "", "parameters": {"type": "object", "properties": {"code": {"type": "string", "description": ""}}, "required": ["code"]}}}] | disabled | + | NousResearch-Hermes-2-Pro-Llama-3-8B-tool_use | 64 | test | {} | [{"type":"function", "function": {"name": "test", "description": "", "parameters": {"type": "object", "properties": {}}}}] | disabled | + | NousResearch-Hermes-2-Pro-Llama-3-8B-tool_use | 128 | ipython | {"code": "Yes,"} | [{"type":"function", "function": {"name": "ipython", "description": "", "parameters": {"type": "object", "properties": {"code": {"type": "string", "description": ""}}, "required": ["code"]}}}] | disabled | + | NousResearch-Hermes-3-Llama-3.1-8B-tool_use | 64 | test | {} | [{"type":"function", "function": {"name": "test", "description": "", "parameters": {"type": "object", "properties": {}}}}] | disabled | + | NousResearch-Hermes-3-Llama-3.1-8B-tool_use | 128 | ipython | {"code": "Yes,"} | [{"type":"function", "function": {"name": "ipython", "description": "", "parameters": {"type": "object", "properties": {"code": {"type": "string", "description": ""}}, "required": ["code"]}}}] | disabled | + | meta-llama-Meta-Llama-3.1-8B-Instruct | 64 | test | {} | [{"type":"function", "function": {"name": "test", "description": "", "parameters": {"type": "object", "properties": {}}}}] | disabled | + | meta-llama-Meta-Llama-3.1-8B-Instruct | 64 | ipython | {"code": "it and realed at the otter. Asked Dave Dasty, Daisy is a big, shiny blue. As"} | [{"type":"function", "function": {"name": "ipython", "description": "", "parameters": {"type": "object", "properties": {"code": {"type": "string", "description": ""}}, "required": ["code"]}}}] | disabled | + | meta-llama-Llama-3.2-3B-Instruct | 64 | test | {} | [{"type":"function", "function": {"name": "test", "description": "", "parameters": {"type": "object", "properties": {}}}}] | disabled | + | meta-llama-Llama-3.2-3B-Instruct | 64 | ipython | {"code": "Yes,"} | [{"type":"function", "function": {"name": "ipython", "description": "", "parameters": {"type": "object", "properties": {"code": {"type": "string", "description": ""}}, "required": ["code"]}}}] | disabled | + | mistralai-Mistral-Nemo-Instruct-2407 | 128 | test | {} | [{"type":"function", "function": {"name": "test", "description": "", "parameters": {"type": "object", "properties": {}}}}] | disabled | + | mistralai-Mistral-Nemo-Instruct-2407 | 128 | ipython | {"code": "It's a small cat."} | [{"type":"function", "function": {"name": "ipython", "description": "", "parameters": {"type": "object", "properties": {"code": {"type": "string", "description": ""}}, "required": ["code"]}}}] | disabled | Scenario Outline: Template + tinystories model yields no tool call @@ -92,12 +96,12 @@ Feature: llama.cpp server | tool_name | tool_arguments | hf_repo | hf_file | template_override | | ipython | {"code": "print('Hello, World!')"} | bartowski/Phi-3.5-mini-instruct-GGUF | Phi-3.5-mini-instruct-Q4_K_M.gguf | | | ipython | {"code": "print('Hello, World!')"} | NousResearch/Hermes-2-Pro-Llama-3-8B-GGUF | Hermes-2-Pro-Llama-3-8B-Q4_K_M.gguf | NousResearch-Hermes-2-Pro-Llama-3-8B-tool_use | + | ipython | {"code": "print('Hello World!')"} | NousResearch/Hermes-3-Llama-3.1-8B-GGUF | Hermes-3-Llama-3.1-8B.Q4_K_M.gguf | NousResearch-Hermes-3-Llama-3.1-8B-tool_use | | ipython | {"code": "print('Hello, World!')"} | bartowski/Mistral-Nemo-Instruct-2407-GGUF | Mistral-Nemo-Instruct-2407-Q4_K_M.gguf | mistralai-Mistral-Nemo-Instruct-2407 | | ipython | {"code": "print('Hello, World!'}"} | lmstudio-community/Llama-3.2-1B-Instruct-GGUF | Llama-3.2-1B-Instruct-Q4_K_M.gguf | meta-llama-Llama-3.2-3B-Instruct | | ipython | {"code": "print("} | lmstudio-community/Llama-3.2-3B-Instruct-GGUF | Llama-3.2-3B-Instruct-Q4_K_M.gguf | meta-llama-Llama-3.2-3B-Instruct | | ipython | {"code": "print("} | lmstudio-community/Meta-Llama-3.1-8B-Instruct-GGUF | Meta-Llama-3.1-8B-Instruct-Q4_K_M.gguf | | | ipython | {"code": "print('Hello, World!')"} | bartowski/functionary-small-v3.2-GGUF | functionary-small-v3.2-Q4_K_M.gguf | meetkai-functionary-medium-v3.2 | - # | ipython | {"code": "print("} | lmstudio-community/Meta-Llama-3.1-70B-Instruct-GGUF | Meta-Llama-3.1-70B-Instruct-Q4_K_M.gguf | | # | ipython | {"code": "print('Hello, world!')"} | bartowski/gemma-2-2b-it-GGUF | gemma-2-2b-it-Q4_K_M.gguf | | diff --git a/examples/server/tests/requirements.txt b/examples/server/tests/requirements.txt index 553954872..a1073ba9d 100644 --- a/examples/server/tests/requirements.txt +++ b/examples/server/tests/requirements.txt @@ -2,6 +2,6 @@ aiohttp~=3.9.3 behave~=1.2.6 huggingface_hub~=0.23.2 numpy~=1.26.4 -openai~=1.30.3 +openai~=1.50.2 prometheus-client~=0.20.0 requests~=2.32.3 diff --git a/scripts/get_hf_chat_template.py b/scripts/get_hf_chat_template.py index 5617309ae..10ae62960 100644 --- a/scripts/get_hf_chat_template.py +++ b/scripts/get_hf_chat_template.py @@ -7,7 +7,7 @@ Examples: python ./scripts/get_hf_chat_template.py NousResearch/Meta-Llama-3-8B-Instruct - python ./scripts/get_hf_chat_template.py NousResearch/Hermes-3-Llama-3.1-70B tool_use + python ./scripts/get_hf_chat_template.py NousResearch/Hermes-3-Llama-3.1-8B tool_use python ./scripts/get_hf_chat_template.py meta-llama/Llama-3.2-3B-Instruct ''' diff --git a/scripts/update_jinja_goldens.py b/scripts/update_jinja_goldens.py index a90adf942..902c0eefe 100644 --- a/scripts/update_jinja_goldens.py +++ b/scripts/update_jinja_goldens.py @@ -47,7 +47,7 @@ model_ids = [ "CohereForAI/c4ai-command-r-plus", "NousResearch/Hermes-2-Pro-Llama-3-8B", "NousResearch/Hermes-2-Pro-Mistral-7B", - "NousResearch/Hermes-3-Llama-3.1-70B", + "NousResearch/Hermes-3-Llama-3.1-8B", "openchat/openchat-3.5-0106", "OrionStarAI/Orion-14B-Chat", "Qwen/Qwen2-7B-Instruct", diff --git a/tests/chat/goldens/NousResearch-Hermes-3-Llama-3.1-70B-default-simple.txt b/tests/chat/goldens/NousResearch-Hermes-3-Llama-3.1-8B-default-simple.txt similarity index 100% rename from tests/chat/goldens/NousResearch-Hermes-3-Llama-3.1-70B-default-simple.txt rename to tests/chat/goldens/NousResearch-Hermes-3-Llama-3.1-8B-default-simple.txt diff --git a/tests/chat/goldens/NousResearch-Hermes-3-Llama-3.1-70B-default-system.txt b/tests/chat/goldens/NousResearch-Hermes-3-Llama-3.1-8B-default-system.txt similarity index 100% rename from tests/chat/goldens/NousResearch-Hermes-3-Llama-3.1-70B-default-system.txt rename to tests/chat/goldens/NousResearch-Hermes-3-Llama-3.1-8B-default-system.txt diff --git a/tests/chat/goldens/NousResearch-Hermes-3-Llama-3.1-70B-tool_use-simple.txt b/tests/chat/goldens/NousResearch-Hermes-3-Llama-3.1-8B-tool_use-simple.txt similarity index 100% rename from tests/chat/goldens/NousResearch-Hermes-3-Llama-3.1-70B-tool_use-simple.txt rename to tests/chat/goldens/NousResearch-Hermes-3-Llama-3.1-8B-tool_use-simple.txt diff --git a/tests/chat/goldens/NousResearch-Hermes-3-Llama-3.1-70B-tool_use-system.txt b/tests/chat/goldens/NousResearch-Hermes-3-Llama-3.1-8B-tool_use-system.txt similarity index 100% rename from tests/chat/goldens/NousResearch-Hermes-3-Llama-3.1-70B-tool_use-system.txt rename to tests/chat/goldens/NousResearch-Hermes-3-Llama-3.1-8B-tool_use-system.txt diff --git a/tests/chat/goldens/NousResearch-Hermes-3-Llama-3.1-70B-tool_use-tool_use.txt b/tests/chat/goldens/NousResearch-Hermes-3-Llama-3.1-8B-tool_use-tool_use.txt similarity index 100% rename from tests/chat/goldens/NousResearch-Hermes-3-Llama-3.1-70B-tool_use-tool_use.txt rename to tests/chat/goldens/NousResearch-Hermes-3-Llama-3.1-8B-tool_use-tool_use.txt diff --git a/tests/chat/templates/NousResearch-Hermes-3-Llama-3.1-70B-default.jinja b/tests/chat/templates/NousResearch-Hermes-3-Llama-3.1-8B-default.jinja similarity index 100% rename from tests/chat/templates/NousResearch-Hermes-3-Llama-3.1-70B-default.jinja rename to tests/chat/templates/NousResearch-Hermes-3-Llama-3.1-8B-default.jinja diff --git a/tests/chat/templates/NousResearch-Hermes-3-Llama-3.1-70B-tool_use.jinja b/tests/chat/templates/NousResearch-Hermes-3-Llama-3.1-8B-tool_use.jinja similarity index 100% rename from tests/chat/templates/NousResearch-Hermes-3-Llama-3.1-70B-tool_use.jinja rename to tests/chat/templates/NousResearch-Hermes-3-Llama-3.1-8B-tool_use.jinja diff --git a/tests/test-tool-call.cpp b/tests/test-tool-call.cpp index b4ecdd7fe..884bbf824 100644 --- a/tests/test-tool-call.cpp +++ b/tests/test-tool-call.cpp @@ -314,6 +314,8 @@ static void test_tool_call_style_detection() { test_tool_call_style("tests/chat/templates/meetkai-functionary-medium-v3.2.jinja", FunctionaryV3Llama3); test_tool_call_style("tests/chat/templates/meta-llama-Meta-Llama-3.1-8B-Instruct.jinja", Llama31); test_tool_call_style("tests/chat/templates/meta-llama-Llama-3.2-3B-Instruct.jinja", Llama32); + test_tool_call_style("tests/chat/templates/NousResearch-Hermes-2-Pro-Llama-3-8B-tool_use.jinja", Hermes2Pro); + test_tool_call_style("tests/chat/templates/NousResearch-Hermes-3-Llama-3.1-8B-tool_use.jinja", Hermes2Pro); test_tool_call_style("tests/chat/templates/CohereForAI-c4ai-command-r-plus-tool_use.jinja", CommandRPlus); test_tool_call_style("tests/chat/templates/mistralai-Mistral-Nemo-Instruct-2407.jinja", MistralNemo); test_tool_call_style("tests/chat/templates/google-gemma-7b-it.jinja", Generic); @@ -395,6 +397,7 @@ static void test_grammars() { test_template("tests/chat/templates/mistralai-Mistral-Nemo-Instruct-2407.jinja", "", "", { "" }, tool_call_message_with_id, tools, /* skip_grammar_test= */ true); test_template("tests/chat/templates/NousResearch-Hermes-2-Pro-Llama-3-8B-tool_use.jinja", "", "", { "<|im_end|>" }, tool_call_message, tools); + test_template("tests/chat/templates/NousResearch-Hermes-3-Llama-3.1-8B-tool_use.jinja", "", "", { "<|im_end|>" }, tool_call_message, tools); test_template("tests/chat/templates/meta-llama-Meta-Llama-3.1-8B-Instruct.jinja", "", "", { "<|eom_id|>", "<|eot_id|>" }, tool_call_message, tools); test_template("tests/chat/templates/meta-llama-Llama-3.2-3B-Instruct.jinja", "", "", { "<|eom_id|>", "<|eot_id|>" }, tool_call_message, tools); test_template("tests/chat/templates/meetkai-functionary-medium-v3.1.jinja", "", "", { "<|eom_id|>", "<|eot_id|>" }, tool_call_message, tools);