From 09de4eb9ed0ee043fdd1a1093138ef3ebcc88662 Mon Sep 17 00:00:00 2001 From: ochafik Date: Sat, 30 Mar 2024 01:57:05 +0000 Subject: [PATCH] openai: actually use thoughtful examples in tests --- examples/openai/test_chat_handlers.md | 51 ++++++++++++++++++--------- examples/openai/test_chat_handlers.py | 4 +-- 2 files changed, 37 insertions(+), 18 deletions(-) diff --git a/examples/openai/test_chat_handlers.md b/examples/openai/test_chat_handlers.md index 28146dac0..a77099892 100644 --- a/examples/openai/test_chat_handlers.md +++ b/examples/openai/test_chat_handlers.md @@ -14,7 +14,7 @@ Messages: "role": "assistant", "name": null, "tool_call_id": null, - "content": null, + "content": "I've thought a lot about this.", "tool_calls": [ { "id": "call_531873", @@ -67,7 +67,7 @@ Prompt: ```js [INST] Add two numbers for the purpose of this test. [/INST]{ - "thought_about_next_step_only": "", + "thought_about_next_step_only": "I've thought a lot about this.", "next_step": { "tool_calls": [ { @@ -239,7 +239,8 @@ thought-about-next-step-only-kv ::= "\"thought_about_next_step_only\"" space ":" Prompt: ```js -[INST] Add two numbers for the purpose of this test. [/INST]{"id": "call_531873", "type": "function", "function": {"name": "superSecretTool", "arguments": {"a": 2535, "b": 32222000403}}}[INST] [TOOL(name=superSecretTool, id=call_531873)]32222002938[/TOOL] [/INST]The sum of 2535 and 32222000403 is 42. +[INST] Add two numbers for the purpose of this test. [/INST]I've thought a lot about this. +{"id": "call_531873", "type": "function", "function": {"name": "superSecretTool", "arguments": {"a": 2535, "b": 32222000403}}}[INST] [TOOL(name=superSecretTool, id=call_531873)]32222002938[/TOOL] [/INST]The sum of 2535 and 32222000403 is 42. ``` @@ -271,7 +272,8 @@ space ::= " "? Prompt: ```js -[INST] Add two numbers for the purpose of this test. [/INST]{"id": "call_531873", "type": "function", "function": {"name": "superSecretTool", "arguments": {"a": 2535, "b": 32222000403}}}[INST] [TOOL(name=superSecretTool, id=call_531873)]32222002938[/TOOL] [/INST]The sum of 2535 and 32222000403 is 42. +[INST] Add two numbers for the purpose of this test. [/INST]I've thought a lot about this. +{"id": "call_531873", "type": "function", "function": {"name": "superSecretTool", "arguments": {"a": 2535, "b": 32222000403}}}[INST] [TOOL(name=superSecretTool, id=call_531873)]32222002938[/TOOL] [/INST]The sum of 2535 and 32222000403 is 42. ``` @@ -364,7 +366,8 @@ tool-call ::= "" space (superSecretTool-tool-call | say-t Prompt: ```js -[INST] Add two numbers for the purpose of this test. [/INST]{"id": "call_531873", "type": "function", "function": {"name": "superSecretTool", "arguments": {"a": 2535, "b": 32222000403}}}[INST] [TOOL(name=superSecretTool, id=call_531873)]32222002938[/TOOL] [/INST]The sum of 2535 and 32222000403 is 42. +[INST] Add two numbers for the purpose of this test. [/INST]I've thought a lot about this. +{"id": "call_531873", "type": "function", "function": {"name": "superSecretTool", "arguments": {"a": 2535, "b": 32222000403}}}[INST] [TOOL(name=superSecretTool, id=call_531873)]32222002938[/TOOL] [/INST]The sum of 2535 and 32222000403 is 42. ``` @@ -446,7 +449,8 @@ Prompt: <|content|>Add two numbers for the purpose of this test. <|from|>assistant <|recipient|>all -<|content|>{"id": "call_531873", "type": "function", "function": {"name": "superSecretTool", "arguments": {"a": 2535, "b": 32222000403}}}<|stop|> +<|content|>I've thought a lot about this. +{"id": "call_531873", "type": "function", "function": {"name": "superSecretTool", "arguments": {"a": 2535, "b": 32222000403}}}<|stop|> <|from|>superSecretTool <|recipient|>all <|content|>32222002938 @@ -517,7 +521,8 @@ Prompt: <|content|>Add two numbers for the purpose of this test. <|from|>assistant <|recipient|>all -<|content|>{"id": "call_531873", "type": "function", "function": {"name": "superSecretTool", "arguments": {"a": 2535, "b": 32222000403}}}<|stop|> +<|content|>I've thought a lot about this. +{"id": "call_531873", "type": "function", "function": {"name": "superSecretTool", "arguments": {"a": 2535, "b": 32222000403}}}<|stop|> <|from|>superSecretTool <|recipient|>all <|content|>32222002938 @@ -573,6 +578,7 @@ Prompt: <|im_start|>user Add two numbers for the purpose of this test.<|im_end|> <|im_start|>assistant +I've thought a lot about this. {"id": "call_531873", "type": "function", "function": {"name": "superSecretTool", "arguments": {"a": 2535, "b": 32222000403}}}<|im_end|> <|im_start|>tool 32222002938<|im_end|> @@ -670,6 +676,7 @@ Prompt: <|im_start|>user Add two numbers for the purpose of this test.<|im_end|> <|im_start|>assistant +I've thought a lot about this. {"id": "call_531873", "type": "function", "function": {"name": "superSecretTool", "arguments": {"a": 2535, "b": 32222000403}}}<|im_end|> <|im_start|>tool 32222002938<|im_end|> @@ -711,6 +718,7 @@ Prompt: <|im_start|>user Add two numbers for the purpose of this test.<|im_end|> <|im_start|>assistant +I've thought a lot about this. {"id": "call_531873", "type": "function", "function": {"name": "superSecretTool", "arguments": {"a": 2535, "b": 32222000403}}}<|im_end|> <|im_start|>tool 32222002938<|im_end|> @@ -813,6 +821,7 @@ Prompt: <|im_start|>user Add two numbers for the purpose of this test.<|im_end|> <|im_start|>assistant +I've thought a lot about this. {"id": "call_531873", "type": "function", "function": {"name": "superSecretTool", "arguments": {"a": 2535, "b": 32222000403}}}<|im_end|> <|im_start|>tool 32222002938<|im_end|> @@ -855,7 +864,7 @@ Prompt: Add two numbers for the purpose of this test.<|im_end|> <|im_start|>assistant { - "thought_about_next_step_only": "", + "thought_about_next_step_only": "I've thought a lot about this.", "next_step": { "tool_calls": [ { @@ -1036,6 +1045,7 @@ Prompt: <|im_start|>user Add two numbers for the purpose of this test.<|im_end|> <|im_start|>assistant +I've thought a lot about this. {"id": "call_531873", "type": "function", "function": {"name": "superSecretTool", "arguments": {"a": 2535, "b": 32222000403}}}<|im_end|> <|im_start|>tool 32222002938<|im_end|> @@ -1077,6 +1087,7 @@ Prompt: <|im_start|>user Add two numbers for the purpose of this test.<|im_end|> <|im_start|>assistant +I've thought a lot about this. {"id": "call_531873", "type": "function", "function": {"name": "superSecretTool", "arguments": {"a": 2535, "b": 32222000403}}}<|im_end|> <|im_start|>tool 32222002938<|im_end|> @@ -1142,6 +1153,7 @@ Prompt: <|im_start|>user Add two numbers for the purpose of this test.<|im_end|> <|im_start|>assistant +I've thought a lot about this. {"id": "call_531873", "type": "function", "function": {"name": "superSecretTool", "arguments": {"a": 2535, "b": 32222000403}}}<|im_end|> <|im_start|>tool 32222002938<|im_end|> @@ -1190,7 +1202,8 @@ Template: Prompt: ```js -[INST] Add two numbers for the purpose of this test. [/INST] {"id": "call_531873", "type": "function", "function": {"name": "superSecretTool", "arguments": {"a": 2535, "b": 32222000403}}} [INST] [TOOL(name=superSecretTool, id=call_531873)]32222002938[/TOOL] [/INST] The sum of 2535 and 32222000403 is 42. +[INST] Add two numbers for the purpose of this test. [/INST] I've thought a lot about this. +{"id": "call_531873", "type": "function", "function": {"name": "superSecretTool", "arguments": {"a": 2535, "b": 32222000403}}} [INST] [TOOL(name=superSecretTool, id=call_531873)]32222002938[/TOOL] [/INST] The sum of 2535 and 32222000403 is 42. ``` @@ -1278,7 +1291,8 @@ tool-call ::= "" space (superSecretTool-tool-call | say-tool-call) s Prompt: ```js -[INST] Add two numbers for the purpose of this test. [/INST] {"id": "call_531873", "type": "function", "function": {"name": "superSecretTool", "arguments": {"a": 2535, "b": 32222000403}}} [INST] [TOOL(name=superSecretTool, id=call_531873)]32222002938[/TOOL] [/INST] The sum of 2535 and 32222000403 is 42. +[INST] Add two numbers for the purpose of this test. [/INST] I've thought a lot about this. +{"id": "call_531873", "type": "function", "function": {"name": "superSecretTool", "arguments": {"a": 2535, "b": 32222000403}}} [INST] [TOOL(name=superSecretTool, id=call_531873)]32222002938[/TOOL] [/INST] The sum of 2535 and 32222000403 is 42. ``` @@ -1310,7 +1324,8 @@ space ::= " "? Prompt: ```js -[INST] Add two numbers for the purpose of this test. [/INST] {"id": "call_531873", "type": "function", "function": {"name": "superSecretTool", "arguments": {"a": 2535, "b": 32222000403}}} [INST] [TOOL(name=superSecretTool, id=call_531873)]32222002938[/TOOL] [/INST] The sum of 2535 and 32222000403 is 42. +[INST] Add two numbers for the purpose of this test. [/INST] I've thought a lot about this. +{"id": "call_531873", "type": "function", "function": {"name": "superSecretTool", "arguments": {"a": 2535, "b": 32222000403}}} [INST] [TOOL(name=superSecretTool, id=call_531873)]32222002938[/TOOL] [/INST] The sum of 2535 and 32222000403 is 42. ``` @@ -1403,7 +1418,8 @@ tool-call ::= "" space (superSecretTool-tool-call | say-tool-call) s Prompt: ```js -[INST] Add two numbers for the purpose of this test. [/INST] {"id": "call_531873", "type": "function", "function": {"name": "superSecretTool", "arguments": {"a": 2535, "b": 32222000403}}} [INST] [TOOL(name=superSecretTool, id=call_531873)]32222002938[/TOOL] [/INST] The sum of 2535 and 32222000403 is 42. +[INST] Add two numbers for the purpose of this test. [/INST] I've thought a lot about this. +{"id": "call_531873", "type": "function", "function": {"name": "superSecretTool", "arguments": {"a": 2535, "b": 32222000403}}} [INST] [TOOL(name=superSecretTool, id=call_531873)]32222002938[/TOOL] [/INST] The sum of 2535 and 32222000403 is 42. ``` @@ -1436,7 +1452,7 @@ Prompt: ```js [INST] Add two numbers for the purpose of this test. [/INST] { - "thought_about_next_step_only": "", + "thought_about_next_step_only": "I've thought a lot about this.", "next_step": { "tool_calls": [ { @@ -1608,7 +1624,8 @@ thought-about-next-step-only-kv ::= "\"thought_about_next_step_only\"" space ":" Prompt: ```js -[INST] Add two numbers for the purpose of this test. [/INST] {"id": "call_531873", "type": "function", "function": {"name": "superSecretTool", "arguments": {"a": 2535, "b": 32222000403}}} [INST] [TOOL(name=superSecretTool, id=call_531873)]32222002938[/TOOL] [/INST] The sum of 2535 and 32222000403 is 42. +[INST] Add two numbers for the purpose of this test. [/INST] I've thought a lot about this. +{"id": "call_531873", "type": "function", "function": {"name": "superSecretTool", "arguments": {"a": 2535, "b": 32222000403}}} [INST] [TOOL(name=superSecretTool, id=call_531873)]32222002938[/TOOL] [/INST] The sum of 2535 and 32222000403 is 42. ``` @@ -1640,7 +1657,8 @@ space ::= " "? Prompt: ```js -[INST] Add two numbers for the purpose of this test. [/INST] {"id": "call_531873", "type": "function", "function": {"name": "superSecretTool", "arguments": {"a": 2535, "b": 32222000403}}} [INST] [TOOL(name=superSecretTool, id=call_531873)]32222002938[/TOOL] [/INST] The sum of 2535 and 32222000403 is 42. +[INST] Add two numbers for the purpose of this test. [/INST] I've thought a lot about this. +{"id": "call_531873", "type": "function", "function": {"name": "superSecretTool", "arguments": {"a": 2535, "b": 32222000403}}} [INST] [TOOL(name=superSecretTool, id=call_531873)]32222002938[/TOOL] [/INST] The sum of 2535 and 32222000403 is 42. ``` @@ -1696,7 +1714,8 @@ tool-call ::= "" space (superSecretTool-tool-call | say-tool-call) s Prompt: ```js -[INST] Add two numbers for the purpose of this test. [/INST] {"id": "call_531873", "type": "function", "function": {"name": "superSecretTool", "arguments": {"a": 2535, "b": 32222000403}}} [INST] [TOOL(name=superSecretTool, id=call_531873)]32222002938[/TOOL] [/INST] The sum of 2535 and 32222000403 is 42. +[INST] Add two numbers for the purpose of this test. [/INST] I've thought a lot about this. +{"id": "call_531873", "type": "function", "function": {"name": "superSecretTool", "arguments": {"a": 2535, "b": 32222000403}}} [INST] [TOOL(name=superSecretTool, id=call_531873)]32222002938[/TOOL] [/INST] The sum of 2535 and 32222000403 is 42. ``` diff --git a/examples/openai/test_chat_handlers.py b/examples/openai/test_chat_handlers.py index fc4b1680b..ee724e002 100644 --- a/examples/openai/test_chat_handlers.py +++ b/examples/openai/test_chat_handlers.py @@ -151,7 +151,7 @@ if __name__ == "__main__": failures = [] - print(f'\nMessages:\n\n```js\n{json.dumps([m.model_dump() for m in TEST_MESSAGES], indent=2)}\n```\n') + print(f'\nMessages:\n\n```js\n{json.dumps([m.model_dump() for m in TEST_MESSAGES_THOUGHT], indent=2)}\n```\n') def check(b: bool, msg: str): if not b: @@ -203,7 +203,7 @@ if __name__ == "__main__": print(f'\n### {model_name} / {style.name} / {tool_situation}\n') - print(f'\nPrompt:\n\n```js\n{ch.render_prompt(TEST_MESSAGES)}\n```\n') + print(f'\nPrompt:\n\n```js\n{ch.render_prompt(TEST_MESSAGES_THOUGHT)}\n```\n') print(f'\nPrompt:\n\n```json\n{ch.output_format_prompt.content}\n```\n')