diff --git a/examples/agent/test_chat_handlers.md b/examples/agent/test_chat_handlers.md new file mode 100644 index 000000000..a93927fca --- /dev/null +++ b/examples/agent/test_chat_handlers.md @@ -0,0 +1,1436 @@ + +Messages: + +```js +[ + { + "role": "user", + "name": null, + "tool_call_id": null, + "content": "What is the sum of 2535 squared and 32222000403 then multiplied by one and a half. What's a third of the result?", + "tool_calls": null + }, + { + "role": "assistant", + "name": null, + "tool_call_id": null, + "content": "?", + "tool_calls": [ + { + "id": "call_531873", + "type": "function", + "function": { + "name": "add", + "arguments": { + "a": 2535, + "b": 32222000403 + } + } + } + ] + }, + { + "role": "tool", + "name": "add", + "tool_call_id": "call_531873", + "content": "32222002938", + "tool_calls": null + } +] +``` + + +# mistral_instruct_v0_1 + + +Template: + +```js +{{ bos_token }}{% for message in messages %}{% if (message['role'] == 'user') != (loop.index0 % 2 == 0) %}{{ raise_exception('Conversation roles must alternate user/assistant/user/assistant/...') }}{% endif %}{% if message['role'] == 'user' %}{{ '[INST] ' + message['content'] + ' [/INST]' }}{% elif message['role'] == 'assistant' %}{{ message['content'] + eos_token}}{% else %}{{ raise_exception('Only user and assistant roles are supported!') }}{% endif %}{% endfor %} +``` + + +Prompt: + +```js +[INST] What is the sum of 2535 squared and 32222000403 then multiplied by one and a half. What's a third of the result? [/INST]? +{"id": "call_531873", "type": "function", "function": {"name": "add", "arguments": {"a": 2535, "b": 32222000403}}}[INST] [TOOL RESULT(name=add, id=call_531873]32222002938[/TOOL RESULT] [/INST] +``` + + +## ToolsPromptStyle.TOOLS_THOUGHTFUL_STEPS + + +### with tools + + +Prompt: + +```json +You are a function calling AI model. +Here are the tools available: +{ + "type": "function", + "function": { + "name": "add", + "description": "Adds two numbers", + "parameters": { + "properties": { + "a": { + "type": "integer" + }, + "b": { + "type": "integer" + } + }, + "required": [ + "a", + "b" + ] + } + } +} +{ + "type": "function", + "function": { + "name": "say", + "description": "Says something out loud (TTS)", + "parameters": { + "properties": { + "text": { + "description": "The text to say out loud", + "type": "string" + } + }, + "required": [ + "text" + ] + } + } +} +Please respond in JSON format with the following schema: { + "type": "object", + "properties": { + "thought_about_next_step_only": { + "title": "Thought about next step", + "type": "string" + }, + "next_step": { + "title": "Next Step: either a result or one or more tool calls to achieve the original goal", + "oneOf": [ + { + "properties": { + "tool_calls": { + "prefixItems": { + "properties": { + "name": { + "title": "Name of the tool to call", + "type": "string" + }, + "arguments": { + "title": "Arguments to pass to the tool", + "type": "object" + } + }, + "required": [ + "name", + "arguments" + ] + } + } + }, + "required": [ + "tool_calls" + ] + }, + { + "title": "Result (achieving original goal)", + "properties": { + "result": { + "type": "integer" + } + }, + "required": [ + "result" + ] + } + ] + } + }, + "required": [ + "original_goal", + "thought_about_next_step_only", + "next_step" + ] +} +``` + + +Grammar: + +```js +decimal-part ::= [0-9] [0-9]? [0-9]? [0-9]? [0-9]? [0-9]? [0-9]? [0-9]? [0-9]? [0-9]? [0-9]? +integer ::= ("-"? integral-part) space +integral-part ::= [0-9] | [1-9] [0-9]? [0-9]? [0-9]? [0-9]? [0-9]? [0-9]? [0-9]? [0-9]? [0-9]? [0-9]? +next-step ::= next-step-0 | next-step-1 +next-step-0 ::= "{" space next-step-0-tool-calls-kv "}" space +next-step-0-tool-calls ::= "[" space ( next-step-0-tool-calls-item ( "," space next-step-0-tool-calls-item )* )? "]" space +next-step-0-tool-calls-item ::= next-step-0-tool-calls-item-0 | next-step-0-tool-calls-item-1 +next-step-0-tool-calls-item-0 ::= "{" space next-step-0-tool-calls-item-0-name-kv "," space next-step-0-tool-calls-item-0-arguments-kv "}" space +next-step-0-tool-calls-item-0-arguments ::= "{" space next-step-0-tool-calls-item-0-arguments-a-kv "," space next-step-0-tool-calls-item-0-arguments-b-kv "}" space +next-step-0-tool-calls-item-0-arguments-a-kv ::= "\"a\"" space ":" space integer +next-step-0-tool-calls-item-0-arguments-b-kv ::= "\"b\"" space ":" space integer +next-step-0-tool-calls-item-0-arguments-kv ::= "\"arguments\"" space ":" space next-step-0-tool-calls-item-0-arguments +next-step-0-tool-calls-item-0-name ::= "\"add\"" +next-step-0-tool-calls-item-0-name-kv ::= "\"name\"" space ":" space next-step-0-tool-calls-item-0-name +next-step-0-tool-calls-item-1 ::= "{" space next-step-0-tool-calls-item-1-name-kv "," space next-step-0-tool-calls-item-1-arguments-kv "}" space +next-step-0-tool-calls-item-1-arguments ::= "{" space next-step-0-tool-calls-item-1-arguments-text-kv "}" space +next-step-0-tool-calls-item-1-arguments-kv ::= "\"arguments\"" space ":" space next-step-0-tool-calls-item-1-arguments +next-step-0-tool-calls-item-1-arguments-text-kv ::= "\"text\"" space ":" space string +next-step-0-tool-calls-item-1-name ::= "\"say\"" +next-step-0-tool-calls-item-1-name-kv ::= "\"name\"" space ":" space next-step-0-tool-calls-item-1-name +next-step-0-tool-calls-kv ::= "\"tool_calls\"" space ":" space next-step-0-tool-calls +next-step-1 ::= "{" space next-step-1-result-kv "}" space +next-step-1-result-kv ::= "\"result\"" space ":" space integer +next-step-kv ::= "\"next_step\"" space ":" space next-step +root ::= "{" space thought-about-next-step-only-kv "," space next-step-kv "}" space +space ::= " "? +string ::= "\"" ( + [^"\\] | + "\\" (["\\/bfnrt] | "u" [0-9a-fA-F] [0-9a-fA-F] [0-9a-fA-F] [0-9a-fA-F]) + )* "\"" space +thought-about-next-step-only-kv ::= "\"thought_about_next_step_only\"" space ":" space string +``` + + +### without tools + + +Prompt: + +```json +Please respond in JSON format with the following schema: { + "type": "integer" +} +``` + + +Grammar: + +```js +decimal-part ::= [0-9] [0-9]? [0-9]? [0-9]? [0-9]? [0-9]? [0-9]? [0-9]? [0-9]? [0-9]? [0-9]? +integral-part ::= [0-9] | [1-9] [0-9]? [0-9]? [0-9]? [0-9]? [0-9]? [0-9]? [0-9]? [0-9]? [0-9]? [0-9]? +root ::= ("-"? integral-part) space +space ::= " "? +``` + + +## ToolsPromptStyle.TOOLS_MIXTRAL + + +### with tools + + +Prompt: + +```json +Call one or more functions to assist with the user query, every time this is possible. Don't make assumptions about what values to plug into functions. Here are the available tools: + +{ + "type": "function", + "function": { + "name": "add", + "description": "Adds two numbers", + "parameters": { + "properties": { + "a": { + "type": "integer" + }, + "b": { + "type": "integer" + } + }, + "required": [ + "a", + "b" + ] + } + } +} +{ + "type": "function", + "function": { + "name": "say", + "description": "Says something out loud (TTS)", + "parameters": { + "properties": { + "text": { + "description": "The text to say out loud", + "type": "string" + } + }, + "required": [ + "text" + ] + } + } +} + + +To call each function, give its name and arguments within XML tags as follows: + +{"name": , "arguments": } + +``` + + +Grammar: + +```js +add-tool-call ::= "{" space add-tool-call-name-kv "," space add-tool-call-arguments-kv "}" space +add-tool-call-arguments ::= "{" space add-tool-call-arguments-a-kv "," space add-tool-call-arguments-b-kv "}" space +add-tool-call-arguments-a-kv ::= "\"a\"" space ":" space integer +add-tool-call-arguments-b-kv ::= "\"b\"" space ":" space integer +add-tool-call-arguments-kv ::= "\"arguments\"" space ":" space add-tool-call-arguments +add-tool-call-name ::= "\"" "add" "\"" space +add-tool-call-name-kv ::= "\"name\"" space ":" space add-tool-call-name +content ::= [^<] | "<" [^t<] | "" space (add-tool-call | say-tool-call) space "" +``` + + +### without tools + + +Prompt: + +```json +Please respond in JSON format with the following schema: { + "type": "integer" +} +``` + + +Grammar: + +```js +decimal-part ::= [0-9] [0-9]? [0-9]? [0-9]? [0-9]? [0-9]? [0-9]? [0-9]? [0-9]? [0-9]? [0-9]? +integral-part ::= [0-9] | [1-9] [0-9]? [0-9]? [0-9]? [0-9]? [0-9]? [0-9]? [0-9]? [0-9]? [0-9]? [0-9]? +root ::= ("-"? integral-part) space +space ::= " "? +``` + + +# functionary_v2_2 + + +Template: + +```js +{#v2.2#} +{% for message in messages %} +{% if message['role'] == 'user' or message['role'] == 'system' %} +{{ '<|from|>' + message['role'] + ' +<|recipient|>all +<|content|>' + message['content'] + ' +' }}{% elif message['role'] == 'tool' %} +{{ '<|from|>' + message['name'] + ' +<|recipient|>all +<|content|>' + message['content'] + ' +' }}{% else %} +{% set contain_content='no'%} +{% if message['content'] is not none %} +{{ '<|from|>assistant +<|recipient|>all +<|content|>' + message['content'] }}{% set contain_content='yes'%} +{% endif %} +{% if 'tool_calls' in message and message['tool_calls'] is not none %} +{% for tool_call in message['tool_calls'] %} +{% set prompt='<|from|>assistant +<|recipient|>' + tool_call['function']['name'] + ' +<|content|>' + tool_call['function']['arguments'] %} +{% if loop.index == 1 and contain_content == "no" %} +{{ prompt }}{% else %} +{{ ' +' + prompt}}{% endif %} +{% endfor %} +{% endif %} +{{ '<|stop|> +' }}{% endif %} +{% endfor %} +{% if add_generation_prompt %}{{ '<|from|>assistant +<|recipient|>' }}{% endif %} +``` + + +Prompt: + +```js +<|from|>user +<|recipient|>all +<|content|>What is the sum of 2535 squared and 32222000403 then multiplied by one and a half. What's a third of the result? +<|from|>assistant +<|recipient|>all +<|content|>? +<|from|>assistant +<|recipient|>add +<|content|>{"a": 2535, "b": 32222000403}<|stop|> +<|from|>add +<|recipient|>all +<|content|>32222002938 +<|from|>assistant +<|recipient|> +``` + + +## ToolsPromptStyle.TYPESCRIPT_FUNCTIONARY_V2 + + +### with tools + + +Prompt: + +```json +// Supported function definitions that should be called when necessary. +namespace functions { +// Adds two numbers +type add = (_: { +a: number, +b: number +}) => any; + +// Says something out loud (TTS) +type say = (_: { +// The text to say out loud +text: string +}) => any; +} // namespace functions +``` + + +Grammar: + +```js +add-args ::= "{" space add-args-a-kv "," space add-args-b-kv "}" space +add-args-a-kv ::= "\"a\"" space ":" space integer +add-args-b-kv ::= "\"b\"" space ":" space integer +add-call ::= "add" "\n<|content|>\n" add-args "\n" +content ::= start content-without-start +content-without-start ::= "all\n<|content|>" not-from* +decimal-part ::= [0-9] [0-9]? [0-9]? [0-9]? [0-9]? [0-9]? [0-9]? [0-9]? [0-9]? [0-9]? [0-9]? +integer ::= ("-"? integral-part) space +integral-part ::= [0-9] | [1-9] [0-9]? [0-9]? [0-9]? [0-9]? [0-9]? [0-9]? [0-9]? [0-9]? [0-9]? [0-9]? +not-from ::= ([^<] | "<" ([^|] | "|" ([^f] | "f" ([^r] | "r" ([^o] | "o" ([^m] | "m" ([^|] | "|" ([^>])?)?)?)?)?)?)?) +root ::= content-without-start content* (tool-call+ content*)? | tool-call-without-start tool-call* content* +say-args ::= "{" space say-args-text-kv "}" space +say-args-text-kv ::= "\"text\"" space ":" space string +say-call ::= "say" "\n<|content|>\n" say-args "\n" +space ::= " "? +start ::= "<|from|>assistant\n<|recipient|>" +string ::= "\"" ( + [^"\\] | + "\\" (["\\/bfnrt] | "u" [0-9a-fA-F] [0-9a-fA-F] [0-9a-fA-F] [0-9a-fA-F]) + )* "\"" space +tool-call ::= start tool-call-without-start +tool-call-without-start ::= add-call | say-call +``` + + +### without tools + + +Prompt: + +```json +Please respond in JSON format with the following schema: { + "type": "integer" +} +``` + + +Grammar: + +```js +decimal-part ::= [0-9] [0-9]? [0-9]? [0-9]? [0-9]? [0-9]? [0-9]? [0-9]? [0-9]? [0-9]? [0-9]? +integral-part ::= [0-9] | [1-9] [0-9]? [0-9]? [0-9]? [0-9]? [0-9]? [0-9]? [0-9]? [0-9]? [0-9]? [0-9]? +root ::= ("-"? integral-part) space +space ::= " "? +``` + + +# hermes_2_pro_mistral + + +Template: + +```js +{% for message in messages %}{{'<|im_start|>' + message['role'] + ' +' + message['content'] + '<|im_end|>' + ' +'}}{% endfor %}{% if add_generation_prompt %}{{ '<|im_start|>assistant +' }}{% endif %} +``` + + +Prompt: + +```js +<|im_start|>user +What is the sum of 2535 squared and 32222000403 then multiplied by one and a half. What's a third of the result?<|im_end|> +<|im_start|>assistant +?<|im_end|> +<|im_start|>tool +32222002938<|im_end|> +<|im_start|>assistant + +``` + + +## ToolsPromptStyle.TOOLS_SHORT + + +### with tools + + +Prompt: + +```json +Here are the tools available: + +{ + "type": "function", + "function": { + "name": "add", + "description": "Adds two numbers", + "parameters": { + "properties": { + "a": { + "type": "integer" + }, + "b": { + "type": "integer" + } + }, + "required": [ + "a", + "b" + ] + } + } +} +{ + "type": "function", + "function": { + "name": "say", + "description": "Says something out loud (TTS)", + "parameters": { + "properties": { + "text": { + "description": "The text to say out loud", + "type": "string" + } + }, + "required": [ + "text" + ] + } + } +} + +``` + + +Grammar: + +```js +add-tool-call ::= "{" space add-tool-call-name-kv "," space add-tool-call-arguments-kv "}" space +add-tool-call-arguments ::= "{" space add-tool-call-arguments-a-kv "," space add-tool-call-arguments-b-kv "}" space +add-tool-call-arguments-a-kv ::= "\"a\"" space ":" space integer +add-tool-call-arguments-b-kv ::= "\"b\"" space ":" space integer +add-tool-call-arguments-kv ::= "\"arguments\"" space ":" space add-tool-call-arguments +add-tool-call-name ::= "\"add\"" +add-tool-call-name-kv ::= "\"name\"" space ":" space add-tool-call-name +content ::= [^<] | "<" [^t<] | "" +``` + + +### without tools + + +Prompt: + +```json +Please respond in JSON format with the following schema: { + "type": "integer" +} +``` + + +Grammar: + +```js +decimal-part ::= [0-9] [0-9]? [0-9]? [0-9]? [0-9]? [0-9]? [0-9]? [0-9]? [0-9]? [0-9]? [0-9]? +integral-part ::= [0-9] | [1-9] [0-9]? [0-9]? [0-9]? [0-9]? [0-9]? [0-9]? [0-9]? [0-9]? [0-9]? [0-9]? +root ::= ("-"? integral-part) space +space ::= " "? +``` + + +## ToolsPromptStyle.TOOLS_LONG + + +### with tools + + +Prompt: + +```json +Call one or more functions to assist with the user query, every time this is possible. Don't make assumptions about what values to plug into functions. Here are the available tools: + +{ + "type": "function", + "function": { + "name": "add", + "description": "Adds two numbers", + "parameters": { + "properties": { + "a": { + "type": "integer" + }, + "b": { + "type": "integer" + } + }, + "required": [ + "a", + "b" + ] + } + } +} +{ + "type": "function", + "function": { + "name": "say", + "description": "Says something out loud (TTS)", + "parameters": { + "properties": { + "text": { + "description": "The text to say out loud", + "type": "string" + } + }, + "required": [ + "text" + ] + } + } +} + + +To call each function, give its name and arguments within XML tags as follows: + +{"name": , "arguments": } + +``` + + +Grammar: + +```js +add-tool-call ::= "{" space add-tool-call-name-kv "," space add-tool-call-arguments-kv "}" space +add-tool-call-arguments ::= "{" space add-tool-call-arguments-a-kv "," space add-tool-call-arguments-b-kv "}" space +add-tool-call-arguments-a-kv ::= "\"a\"" space ":" space integer +add-tool-call-arguments-b-kv ::= "\"b\"" space ":" space integer +add-tool-call-arguments-kv ::= "\"arguments\"" space ":" space add-tool-call-arguments +add-tool-call-name ::= "\"add\"" +add-tool-call-name-kv ::= "\"name\"" space ":" space add-tool-call-name +content ::= [^<] | "<" [^t<] | "" +``` + + +### without tools + + +Prompt: + +```json +Please respond in JSON format with the following schema: { + "type": "integer" +} +``` + + +Grammar: + +```js +decimal-part ::= [0-9] [0-9]? [0-9]? [0-9]? [0-9]? [0-9]? [0-9]? [0-9]? [0-9]? [0-9]? [0-9]? +integral-part ::= [0-9] | [1-9] [0-9]? [0-9]? [0-9]? [0-9]? [0-9]? [0-9]? [0-9]? [0-9]? [0-9]? [0-9]? +root ::= ("-"? integral-part) space +space ::= " "? +``` + + +## ToolsPromptStyle.TOOLS_THOUGHTFUL_STEPS + + +### with tools + + +Prompt: + +```json +You are a function calling AI model. +Here are the tools available: +{ + "type": "function", + "function": { + "name": "add", + "description": "Adds two numbers", + "parameters": { + "properties": { + "a": { + "type": "integer" + }, + "b": { + "type": "integer" + } + }, + "required": [ + "a", + "b" + ] + } + } +} +{ + "type": "function", + "function": { + "name": "say", + "description": "Says something out loud (TTS)", + "parameters": { + "properties": { + "text": { + "description": "The text to say out loud", + "type": "string" + } + }, + "required": [ + "text" + ] + } + } +} +Please respond in JSON format with the following schema: { + "type": "object", + "properties": { + "thought_about_next_step_only": { + "title": "Thought about next step", + "type": "string" + }, + "next_step": { + "title": "Next Step: either a result or one or more tool calls to achieve the original goal", + "oneOf": [ + { + "properties": { + "tool_calls": { + "prefixItems": { + "properties": { + "name": { + "title": "Name of the tool to call", + "type": "string" + }, + "arguments": { + "title": "Arguments to pass to the tool", + "type": "object" + } + }, + "required": [ + "name", + "arguments" + ] + } + } + }, + "required": [ + "tool_calls" + ] + }, + { + "title": "Result (achieving original goal)", + "properties": { + "result": { + "type": "integer" + } + }, + "required": [ + "result" + ] + } + ] + } + }, + "required": [ + "original_goal", + "thought_about_next_step_only", + "next_step" + ] +} +``` + + +Grammar: + +```js +decimal-part ::= [0-9] [0-9]? [0-9]? [0-9]? [0-9]? [0-9]? [0-9]? [0-9]? [0-9]? [0-9]? [0-9]? +integer ::= ("-"? integral-part) space +integral-part ::= [0-9] | [1-9] [0-9]? [0-9]? [0-9]? [0-9]? [0-9]? [0-9]? [0-9]? [0-9]? [0-9]? [0-9]? +next-step ::= next-step-0 | next-step-1 +next-step-0 ::= "{" space next-step-0-tool-calls-kv "}" space +next-step-0-tool-calls ::= "[" space ( next-step-0-tool-calls-item ( "," space next-step-0-tool-calls-item )* )? "]" space +next-step-0-tool-calls-item ::= next-step-0-tool-calls-item-0 | next-step-0-tool-calls-item-1 +next-step-0-tool-calls-item-0 ::= "{" space next-step-0-tool-calls-item-0-name-kv "," space next-step-0-tool-calls-item-0-arguments-kv "}" space +next-step-0-tool-calls-item-0-arguments ::= "{" space next-step-0-tool-calls-item-0-arguments-a-kv "," space next-step-0-tool-calls-item-0-arguments-b-kv "}" space +next-step-0-tool-calls-item-0-arguments-a-kv ::= "\"a\"" space ":" space integer +next-step-0-tool-calls-item-0-arguments-b-kv ::= "\"b\"" space ":" space integer +next-step-0-tool-calls-item-0-arguments-kv ::= "\"arguments\"" space ":" space next-step-0-tool-calls-item-0-arguments +next-step-0-tool-calls-item-0-name ::= "\"add\"" +next-step-0-tool-calls-item-0-name-kv ::= "\"name\"" space ":" space next-step-0-tool-calls-item-0-name +next-step-0-tool-calls-item-1 ::= "{" space next-step-0-tool-calls-item-1-name-kv "," space next-step-0-tool-calls-item-1-arguments-kv "}" space +next-step-0-tool-calls-item-1-arguments ::= "{" space next-step-0-tool-calls-item-1-arguments-text-kv "}" space +next-step-0-tool-calls-item-1-arguments-kv ::= "\"arguments\"" space ":" space next-step-0-tool-calls-item-1-arguments +next-step-0-tool-calls-item-1-arguments-text-kv ::= "\"text\"" space ":" space string +next-step-0-tool-calls-item-1-name ::= "\"say\"" +next-step-0-tool-calls-item-1-name-kv ::= "\"name\"" space ":" space next-step-0-tool-calls-item-1-name +next-step-0-tool-calls-kv ::= "\"tool_calls\"" space ":" space next-step-0-tool-calls +next-step-1 ::= "{" space next-step-1-result-kv "}" space +next-step-1-result-kv ::= "\"result\"" space ":" space integer +next-step-kv ::= "\"next_step\"" space ":" space next-step +root ::= "{" space thought-about-next-step-only-kv "," space next-step-kv "}" space +space ::= " "? +string ::= "\"" ( + [^"\\] | + "\\" (["\\/bfnrt] | "u" [0-9a-fA-F] [0-9a-fA-F] [0-9a-fA-F] [0-9a-fA-F]) + )* "\"" space +thought-about-next-step-only-kv ::= "\"thought_about_next_step_only\"" space ":" space string +``` + + +### without tools + + +Prompt: + +```json +Please respond in JSON format with the following schema: { + "type": "integer" +} +``` + + +Grammar: + +```js +decimal-part ::= [0-9] [0-9]? [0-9]? [0-9]? [0-9]? [0-9]? [0-9]? [0-9]? [0-9]? [0-9]? [0-9]? +integral-part ::= [0-9] | [1-9] [0-9]? [0-9]? [0-9]? [0-9]? [0-9]? [0-9]? [0-9]? [0-9]? [0-9]? [0-9]? +root ::= ("-"? integral-part) space +space ::= " "? +``` + + +## ToolsPromptStyle.TOOLS_HERMES_2_PRO + + +### with tools + + +Prompt: + +```json +You are a function calling AI agent with self-recursion. You can call only one function at a time and analyse data you get from function response. You are provided with function signatures within XML tags. The current date is: 2024-03-29. You may use agentic frameworks for reasoning and planning to help with user query. Please call a function and wait for function results to be provided to you in the next iteration. Don't make assumptions about what values to plug into function arguments. Once you have called a function, results will be fed back to you within XML tags. Don't make assumptions about tool results if XML tags are not present since function hasn't been executed yet. Analyze the data once you get the results and call another function. At each iteration please continue adding the your analysis to previous summary. Your final response should directly answer the user query with an anlysis or summary of the results of function calls. Here are the available tools: ['{"type":"function","function":{"name":"add","description":"Adds two numbers","parameters":{"properties":{"a":{"type":"integer"},"b":{"type":"integer"}},"required":["a","b"]}}}', '{"type":"function","function":{"name":"say","description":"Says something out loud (TTS)","parameters":{"properties":{"text":{"description":"The text to say out loud","type":"string"}},"required":["text"]}}}'] If the provided function signatures doesn't have the function you must call, you may write executable python code in markdown syntax and call code_interpreter() function as follows: {"arguments": {"code_markdown": , "name": "code_interpreter"}} Make sure that the json object above with code markdown block is parseable with json.loads() and the XML block with XML ElementTree. Use the following pydantic model json schema for each tool call you will make: {'properties': {'arguments': {'title': 'Arguments', 'type': 'object'}, 'name': {'title': 'Name', 'type': 'string'}}, 'required': ['arguments', 'name'], 'title': 'FunctionCall', 'type': 'object'} At the very first turn you don't have so you shouldn't not make up the results. +Please keep a running summary with analysis of previous function results and summaries from previous iterations. +Do not stop calling functions until the task has been accomplished or you've reached max iteration of 10. +Calling multiple functions at once can overload the system and increase cost so call one function at a time please. +If you plan to continue with analysis, always call another function. +For each function call return a valid json object (using doulbe quotes) with function name and arguments within XML tags as follows: + +{"arguments": , "name": } + + +``` + + +Grammar: + +```js +add-tool-call ::= "{" space add-tool-call-name-kv "," space add-tool-call-arguments-kv "}" space +add-tool-call-arguments ::= "{" space add-tool-call-arguments-a-kv "," space add-tool-call-arguments-b-kv "}" space +add-tool-call-arguments-a-kv ::= "\"a\"" space ":" space integer +add-tool-call-arguments-b-kv ::= "\"b\"" space ":" space integer +add-tool-call-arguments-kv ::= "\"arguments\"" space ":" space add-tool-call-arguments +add-tool-call-name ::= "\"add\"" +add-tool-call-name-kv ::= "\"name\"" space ":" space add-tool-call-name +content ::= [^<] | "<" [^t<] | "" +``` + + +### without tools + + +Prompt: + +```json +Please respond in JSON format with the following schema: { + "type": "integer" +} +``` + + +Grammar: + +```js +decimal-part ::= [0-9] [0-9]? [0-9]? [0-9]? [0-9]? [0-9]? [0-9]? [0-9]? [0-9]? [0-9]? [0-9]? +integral-part ::= [0-9] | [1-9] [0-9]? [0-9]? [0-9]? [0-9]? [0-9]? [0-9]? [0-9]? [0-9]? [0-9]? [0-9]? +root ::= ("-"? integral-part) space +space ::= " "? +``` + + +# llama2 + + +Template: + +```js +{% if messages[0]['role'] == 'system' %}{% set loop_messages = messages[1:] %}{% set system_message = messages[0]['content'] %}{% else %}{% set loop_messages = messages %}{% set system_message = false %}{% endif %}{% for message in loop_messages %}{% if (message['role'] == 'user') != (loop.index0 % 2 == 0) %}{{ raise_exception('Conversation roles must alternate user/assistant/user/assistant/...') }}{% endif %}{% if loop.index0 == 0 and system_message != false %}{% set content = '<>\n' + system_message + '\n<>\n\n' + message['content'] %}{% else %}{% set content = message['content'] %}{% endif %}{% if message['role'] == 'user' %}{{ bos_token + '[INST] ' + content.strip() + ' [/INST]' }}{% elif message['role'] == 'assistant' %}{{ ' ' + content.strip() + ' ' + eos_token }}{% endif %}{% endfor %} +``` + + +Prompt: + +```js +[INST] What is the sum of 2535 squared and 32222000403 then multiplied by one and a half. What's a third of the result? [/INST] ? +{"id": "call_531873", "type": "function", "function": {"name": "add", "arguments": {"a": 2535, "b": 32222000403}}} [INST] [TOOL RESULT(name=add, id=call_531873]32222002938[/TOOL RESULT] [/INST] +``` + + +## ToolsPromptStyle.TOOLS_SHORT + + +### with tools + + +Prompt: + +```json +Here are the tools available: + +{ + "type": "function", + "function": { + "name": "add", + "description": "Adds two numbers", + "parameters": { + "properties": { + "a": { + "type": "integer" + }, + "b": { + "type": "integer" + } + }, + "required": [ + "a", + "b" + ] + } + } +} +{ + "type": "function", + "function": { + "name": "say", + "description": "Says something out loud (TTS)", + "parameters": { + "properties": { + "text": { + "description": "The text to say out loud", + "type": "string" + } + }, + "required": [ + "text" + ] + } + } +} + +``` + + +Grammar: + +```js +add-tool-call ::= "{" space add-tool-call-name-kv "," space add-tool-call-arguments-kv "}" space +add-tool-call-arguments ::= "{" space add-tool-call-arguments-a-kv "," space add-tool-call-arguments-b-kv "}" space +add-tool-call-arguments-a-kv ::= "\"a\"" space ":" space integer +add-tool-call-arguments-b-kv ::= "\"b\"" space ":" space integer +add-tool-call-arguments-kv ::= "\"arguments\"" space ":" space add-tool-call-arguments +add-tool-call-name ::= "\"add\"" +add-tool-call-name-kv ::= "\"name\"" space ":" space add-tool-call-name +content ::= [^<] | "<" [^t<] | "" +``` + + +### without tools + + +Prompt: + +```json +Please respond in JSON format with the following schema: { + "type": "integer" +} +``` + + +Grammar: + +```js +decimal-part ::= [0-9] [0-9]? [0-9]? [0-9]? [0-9]? [0-9]? [0-9]? [0-9]? [0-9]? [0-9]? [0-9]? +integral-part ::= [0-9] | [1-9] [0-9]? [0-9]? [0-9]? [0-9]? [0-9]? [0-9]? [0-9]? [0-9]? [0-9]? [0-9]? +root ::= ("-"? integral-part) space +space ::= " "? +``` + + +## ToolsPromptStyle.TOOLS_LONG + + +### with tools + + +Prompt: + +```json +Call one or more functions to assist with the user query, every time this is possible. Don't make assumptions about what values to plug into functions. Here are the available tools: + +{ + "type": "function", + "function": { + "name": "add", + "description": "Adds two numbers", + "parameters": { + "properties": { + "a": { + "type": "integer" + }, + "b": { + "type": "integer" + } + }, + "required": [ + "a", + "b" + ] + } + } +} +{ + "type": "function", + "function": { + "name": "say", + "description": "Says something out loud (TTS)", + "parameters": { + "properties": { + "text": { + "description": "The text to say out loud", + "type": "string" + } + }, + "required": [ + "text" + ] + } + } +} + + +To call each function, give its name and arguments within XML tags as follows: + +{"name": , "arguments": } + +``` + + +Grammar: + +```js +add-tool-call ::= "{" space add-tool-call-name-kv "," space add-tool-call-arguments-kv "}" space +add-tool-call-arguments ::= "{" space add-tool-call-arguments-a-kv "," space add-tool-call-arguments-b-kv "}" space +add-tool-call-arguments-a-kv ::= "\"a\"" space ":" space integer +add-tool-call-arguments-b-kv ::= "\"b\"" space ":" space integer +add-tool-call-arguments-kv ::= "\"arguments\"" space ":" space add-tool-call-arguments +add-tool-call-name ::= "\"add\"" +add-tool-call-name-kv ::= "\"name\"" space ":" space add-tool-call-name +content ::= [^<] | "<" [^t<] | "" +``` + + +### without tools + + +Prompt: + +```json +Please respond in JSON format with the following schema: { + "type": "integer" +} +``` + + +Grammar: + +```js +decimal-part ::= [0-9] [0-9]? [0-9]? [0-9]? [0-9]? [0-9]? [0-9]? [0-9]? [0-9]? [0-9]? [0-9]? +integral-part ::= [0-9] | [1-9] [0-9]? [0-9]? [0-9]? [0-9]? [0-9]? [0-9]? [0-9]? [0-9]? [0-9]? [0-9]? +root ::= ("-"? integral-part) space +space ::= " "? +``` + + +## ToolsPromptStyle.TOOLS_THOUGHTFUL_STEPS + + +### with tools + + +Prompt: + +```json +You are a function calling AI model. +Here are the tools available: +{ + "type": "function", + "function": { + "name": "add", + "description": "Adds two numbers", + "parameters": { + "properties": { + "a": { + "type": "integer" + }, + "b": { + "type": "integer" + } + }, + "required": [ + "a", + "b" + ] + } + } +} +{ + "type": "function", + "function": { + "name": "say", + "description": "Says something out loud (TTS)", + "parameters": { + "properties": { + "text": { + "description": "The text to say out loud", + "type": "string" + } + }, + "required": [ + "text" + ] + } + } +} +Please respond in JSON format with the following schema: { + "type": "object", + "properties": { + "thought_about_next_step_only": { + "title": "Thought about next step", + "type": "string" + }, + "next_step": { + "title": "Next Step: either a result or one or more tool calls to achieve the original goal", + "oneOf": [ + { + "properties": { + "tool_calls": { + "prefixItems": { + "properties": { + "name": { + "title": "Name of the tool to call", + "type": "string" + }, + "arguments": { + "title": "Arguments to pass to the tool", + "type": "object" + } + }, + "required": [ + "name", + "arguments" + ] + } + } + }, + "required": [ + "tool_calls" + ] + }, + { + "title": "Result (achieving original goal)", + "properties": { + "result": { + "type": "integer" + } + }, + "required": [ + "result" + ] + } + ] + } + }, + "required": [ + "original_goal", + "thought_about_next_step_only", + "next_step" + ] +} +``` + + +Grammar: + +```js +decimal-part ::= [0-9] [0-9]? [0-9]? [0-9]? [0-9]? [0-9]? [0-9]? [0-9]? [0-9]? [0-9]? [0-9]? +integer ::= ("-"? integral-part) space +integral-part ::= [0-9] | [1-9] [0-9]? [0-9]? [0-9]? [0-9]? [0-9]? [0-9]? [0-9]? [0-9]? [0-9]? [0-9]? +next-step ::= next-step-0 | next-step-1 +next-step-0 ::= "{" space next-step-0-tool-calls-kv "}" space +next-step-0-tool-calls ::= "[" space ( next-step-0-tool-calls-item ( "," space next-step-0-tool-calls-item )* )? "]" space +next-step-0-tool-calls-item ::= next-step-0-tool-calls-item-0 | next-step-0-tool-calls-item-1 +next-step-0-tool-calls-item-0 ::= "{" space next-step-0-tool-calls-item-0-name-kv "," space next-step-0-tool-calls-item-0-arguments-kv "}" space +next-step-0-tool-calls-item-0-arguments ::= "{" space next-step-0-tool-calls-item-0-arguments-a-kv "," space next-step-0-tool-calls-item-0-arguments-b-kv "}" space +next-step-0-tool-calls-item-0-arguments-a-kv ::= "\"a\"" space ":" space integer +next-step-0-tool-calls-item-0-arguments-b-kv ::= "\"b\"" space ":" space integer +next-step-0-tool-calls-item-0-arguments-kv ::= "\"arguments\"" space ":" space next-step-0-tool-calls-item-0-arguments +next-step-0-tool-calls-item-0-name ::= "\"add\"" +next-step-0-tool-calls-item-0-name-kv ::= "\"name\"" space ":" space next-step-0-tool-calls-item-0-name +next-step-0-tool-calls-item-1 ::= "{" space next-step-0-tool-calls-item-1-name-kv "," space next-step-0-tool-calls-item-1-arguments-kv "}" space +next-step-0-tool-calls-item-1-arguments ::= "{" space next-step-0-tool-calls-item-1-arguments-text-kv "}" space +next-step-0-tool-calls-item-1-arguments-kv ::= "\"arguments\"" space ":" space next-step-0-tool-calls-item-1-arguments +next-step-0-tool-calls-item-1-arguments-text-kv ::= "\"text\"" space ":" space string +next-step-0-tool-calls-item-1-name ::= "\"say\"" +next-step-0-tool-calls-item-1-name-kv ::= "\"name\"" space ":" space next-step-0-tool-calls-item-1-name +next-step-0-tool-calls-kv ::= "\"tool_calls\"" space ":" space next-step-0-tool-calls +next-step-1 ::= "{" space next-step-1-result-kv "}" space +next-step-1-result-kv ::= "\"result\"" space ":" space integer +next-step-kv ::= "\"next_step\"" space ":" space next-step +root ::= "{" space thought-about-next-step-only-kv "," space next-step-kv "}" space +space ::= " "? +string ::= "\"" ( + [^"\\] | + "\\" (["\\/bfnrt] | "u" [0-9a-fA-F] [0-9a-fA-F] [0-9a-fA-F] [0-9a-fA-F]) + )* "\"" space +thought-about-next-step-only-kv ::= "\"thought_about_next_step_only\"" space ":" space string +``` + + +### without tools + + +Prompt: + +```json +Please respond in JSON format with the following schema: { + "type": "integer" +} +``` + + +Grammar: + +```js +decimal-part ::= [0-9] [0-9]? [0-9]? [0-9]? [0-9]? [0-9]? [0-9]? [0-9]? [0-9]? [0-9]? [0-9]? +integral-part ::= [0-9] | [1-9] [0-9]? [0-9]? [0-9]? [0-9]? [0-9]? [0-9]? [0-9]? [0-9]? [0-9]? [0-9]? +root ::= ("-"? integral-part) space +space ::= " "? +``` + + +## ToolsPromptStyle.TOOLS_HERMES_2_PRO + + +### with tools + + +Prompt: + +```json +You are a function calling AI agent with self-recursion. You can call only one function at a time and analyse data you get from function response. You are provided with function signatures within XML tags. The current date is: 2024-03-29. You may use agentic frameworks for reasoning and planning to help with user query. Please call a function and wait for function results to be provided to you in the next iteration. Don't make assumptions about what values to plug into function arguments. Once you have called a function, results will be fed back to you within XML tags. Don't make assumptions about tool results if XML tags are not present since function hasn't been executed yet. Analyze the data once you get the results and call another function. At each iteration please continue adding the your analysis to previous summary. Your final response should directly answer the user query with an anlysis or summary of the results of function calls. Here are the available tools: ['{"type":"function","function":{"name":"add","description":"Adds two numbers","parameters":{"properties":{"a":{"type":"integer"},"b":{"type":"integer"}},"required":["a","b"]}}}', '{"type":"function","function":{"name":"say","description":"Says something out loud (TTS)","parameters":{"properties":{"text":{"description":"The text to say out loud","type":"string"}},"required":["text"]}}}'] If the provided function signatures doesn't have the function you must call, you may write executable python code in markdown syntax and call code_interpreter() function as follows: {"arguments": {"code_markdown": , "name": "code_interpreter"}} Make sure that the json object above with code markdown block is parseable with json.loads() and the XML block with XML ElementTree. Use the following pydantic model json schema for each tool call you will make: {'properties': {'arguments': {'title': 'Arguments', 'type': 'object'}, 'name': {'title': 'Name', 'type': 'string'}}, 'required': ['arguments', 'name'], 'title': 'FunctionCall', 'type': 'object'} At the very first turn you don't have so you shouldn't not make up the results. +Please keep a running summary with analysis of previous function results and summaries from previous iterations. +Do not stop calling functions until the task has been accomplished or you've reached max iteration of 10. +Calling multiple functions at once can overload the system and increase cost so call one function at a time please. +If you plan to continue with analysis, always call another function. +For each function call return a valid json object (using doulbe quotes) with function name and arguments within XML tags as follows: + +{"arguments": , "name": } + + +``` + + +Grammar: + +```js +add-tool-call ::= "{" space add-tool-call-name-kv "," space add-tool-call-arguments-kv "}" space +add-tool-call-arguments ::= "{" space add-tool-call-arguments-a-kv "," space add-tool-call-arguments-b-kv "}" space +add-tool-call-arguments-a-kv ::= "\"a\"" space ":" space integer +add-tool-call-arguments-b-kv ::= "\"b\"" space ":" space integer +add-tool-call-arguments-kv ::= "\"arguments\"" space ":" space add-tool-call-arguments +add-tool-call-name ::= "\"add\"" +add-tool-call-name-kv ::= "\"name\"" space ":" space add-tool-call-name +content ::= [^<] | "<" [^t<] | "" +``` + + +### without tools + + +Prompt: + +```json +Please respond in JSON format with the following schema: { + "type": "integer" +} +``` + + +Grammar: + +```js +decimal-part ::= [0-9] [0-9]? [0-9]? [0-9]? [0-9]? [0-9]? [0-9]? [0-9]? [0-9]? [0-9]? [0-9]? +integral-part ::= [0-9] | [1-9] [0-9]? [0-9]? [0-9]? [0-9]? [0-9]? [0-9]? [0-9]? [0-9]? [0-9]? [0-9]? +root ::= ("-"? integral-part) space +space ::= " "? +``` + diff --git a/examples/agent/test_chat_handlers.py b/examples/agent/test_chat_handlers.py new file mode 100644 index 000000000..f1b212f0e --- /dev/null +++ b/examples/agent/test_chat_handlers.py @@ -0,0 +1,199 @@ +# +# +# python -m examples.agent.test_chat_handlers | tee examples/agent/test_chat_handlers.md + +import json +from pathlib import Path +import typer +from typing import Annotated + +from examples.openai.api import ChatCompletionRequest, ChatCompletionResponse, Message, Tool, ToolFunction +from examples.openai.gguf_kvs import GGUFKeyValues, Keys +from examples.openai.prompting import ChatHandlerArgs, ChatTemplate, ToolsPromptStyle, get_chat_handler + + + +TEST_MESSAGES = [ + Message(**{ + "role": "user", + "name": None, + "tool_call_id": None, + "content": "What is the sum of 2535 squared and 32222000403 then multiplied by one and a half. What's a third of the result?", + "tool_calls": None + }), + Message(**{ + "role": "assistant", + "name": None, + "tool_call_id": None, + "content": "?", + "tool_calls": [ + { + "id": "call_531873", + "type": "function", + "function": { + "name": "add", + "arguments": { + "a": 2535, + "b": 32222000403 + } + } + } + ] + }), + Message(**{ + "role": "tool", + "name": "add", + "tool_call_id": "call_531873", + "content": "32222002938", + "tool_calls": None + }) +] + +TEST_TOOLS = [ + Tool( + type="function", + function=ToolFunction( + name="add", + description="Adds two numbers", + parameters={ + "properties": { + "a": {"type": "integer"}, + "b": {"type": "integer"}, + }, + "required": ["a", "b"] + } + ) + ), + Tool( + type="function", + function=ToolFunction( + name="say", + description="Says something out loud (TTS)", + parameters={ + "properties": { + "text": { + "description": "The text to say out loud", + "type": "string" + }, + }, + "required": ["text"] + } + ) + ) +] + +TEST_OUTPUT_SCHEMA = {"type": "integer"} + +if __name__ == "__main__": + + # chat_templates = { + # 'mistral_instruct_v0_1': ChatTemplate.from_huggingface("mistralai/Mixtral-8x7B-Instruct-v0.1"), + # 'functionary_v2_2': ChatTemplate.from_huggingface("meetkai/functionary-small-v2.2"), + # 'hermes_2_pro_mistral': ChatTemplate.from_huggingface("NousResearch/Hermes-2-Pro-Mistral-7B"), + # 'llama2': ChatTemplate.from_huggingface("meta-llama/Llama-2-7b-chat-hf"), + # } + # print(json.dumps({k: v.model_dump() for k, v in chat_templates.items()}, indent=2)) + # exit(0) + + chat_templates = { + "mistral_instruct_v0_1": { + "template": "{{ bos_token }}{% for message in messages %}{% if (message['role'] == 'user') != (loop.index0 % 2 == 0) %}{{ raise_exception('Conversation roles must alternate user/assistant/user/assistant/...') }}{% endif %}{% if message['role'] == 'user' %}{{ '[INST] ' + message['content'] + ' [/INST]' }}{% elif message['role'] == 'assistant' %}{{ message['content'] + eos_token}}{% else %}{{ raise_exception('Only user and assistant roles are supported!') }}{% endif %}{% endfor %}", + "eos_token": "", + "bos_token": "" + }, + "functionary_v2_2": { + "template": "{#v2.2#}\n{% for message in messages %}\n{% if message['role'] == 'user' or message['role'] == 'system' %}\n{{ '<|from|>' + message['role'] + '\n<|recipient|>all\n<|content|>' + message['content'] + '\n' }}{% elif message['role'] == 'tool' %}\n{{ '<|from|>' + message['name'] + '\n<|recipient|>all\n<|content|>' + message['content'] + '\n' }}{% else %}\n{% set contain_content='no'%}\n{% if message['content'] is not none %}\n{{ '<|from|>assistant\n<|recipient|>all\n<|content|>' + message['content'] }}{% set contain_content='yes'%}\n{% endif %}\n{% if 'tool_calls' in message and message['tool_calls'] is not none %}\n{% for tool_call in message['tool_calls'] %}\n{% set prompt='<|from|>assistant\n<|recipient|>' + tool_call['function']['name'] + '\n<|content|>' + tool_call['function']['arguments'] %}\n{% if loop.index == 1 and contain_content == \"no\" %}\n{{ prompt }}{% else %}\n{{ '\n' + prompt}}{% endif %}\n{% endfor %}\n{% endif %}\n{{ '<|stop|>\n' }}{% endif %}\n{% endfor %}\n{% if add_generation_prompt %}{{ '<|from|>assistant\n<|recipient|>' }}{% endif %}", + "eos_token": "", + "bos_token": "" + }, + "hermes_2_pro_mistral": { + "template": "{% for message in messages %}{{'<|im_start|>' + message['role'] + '\n' + message['content'] + '<|im_end|>' + '\n'}}{% endfor %}{% if add_generation_prompt %}{{ '<|im_start|>assistant\n' }}{% endif %}", + "eos_token": "<|im_end|>", + "bos_token": "" + }, + "llama2": { + "template": "{% if messages[0]['role'] == 'system' %}{% set loop_messages = messages[1:] %}{% set system_message = messages[0]['content'] %}{% else %}{% set loop_messages = messages %}{% set system_message = false %}{% endif %}{% for message in loop_messages %}{% if (message['role'] == 'user') != (loop.index0 % 2 == 0) %}{{ raise_exception('Conversation roles must alternate user/assistant/user/assistant/...') }}{% endif %}{% if loop.index0 == 0 and system_message != false %}{% set content = '<>\\n' + system_message + '\\n<>\\n\\n' + message['content'] %}{% else %}{% set content = message['content'] %}{% endif %}{% if message['role'] == 'user' %}{{ bos_token + '[INST] ' + content.strip() + ' [/INST]' }}{% elif message['role'] == 'assistant' %}{{ ' ' + content.strip() + ' ' + eos_token }}{% endif %}{% endfor %}", + "eos_token": "", + "bos_token": "" + }, + } + chat_templates = {k: ChatTemplate(**v) for k, v in chat_templates.items()} + + print(f'\nMessages:\n\n```js\n{json.dumps([m.model_dump() for m in TEST_MESSAGES], indent=2)}\n```\n') + + for model_name, chat_template in chat_templates.items(): + print(f"\n# {model_name}\n") + print(f'\nTemplate:\n\n```js\n{chat_template.template}\n```\n') + + print(f'\nPrompt:\n\n```js\n{chat_template.render(TEST_MESSAGES, add_generation_prompt=True)}\n```\n') + + argss = { + "with tools": ChatHandlerArgs( + chat_template=chat_template, #ChatTemplate.from_gguf(GGUFKeyValues(model)), + response_schema=TEST_OUTPUT_SCHEMA, + tools=TEST_TOOLS, + ), + "without tools": ChatHandlerArgs( + chat_template=chat_template, #ChatTemplate.from_gguf(GGUFKeyValues(model)), + response_schema=TEST_OUTPUT_SCHEMA, + tools=[], + ), + } + + for style in ToolsPromptStyle: + if (style == ToolsPromptStyle.TYPESCRIPT_FUNCTIONARY_V2) != (model_name.startswith("functionary")): + continue + + if style == ToolsPromptStyle.TOOLS_MIXTRAL and model_name != "mistral_instruct_v0_1": + continue + + if model_name == "mistral_instruct_v0_1" and style not in (ToolsPromptStyle.TOOLS_THOUGHTFUL_STEPS, ToolsPromptStyle.TOOLS_MIXTRAL): + continue + + print(f'\n## {style}\n') + + for tn, args in argss.items(): + ch = get_chat_handler(args, parallel_calls=True, tool_style=style) + + print(f'\n### {tn}\n') + + print(f'\nPrompt:\n\n```json\n{ch.output_format_prompt.content}\n```\n') + + print(f'\nGrammar:\n\n```js\n{ch.grammar}\n```\n') + + + # test_templates([ + # Message(**{ + # "role": "user", + # "name": None, + # "tool_call_id": None, + # "content": "What is the sum of 2535 squared and 32222000403 then multiplied by one and a half. What's a third of the result?", + # "tool_calls": None + # }), + # Message(**{ + # "role": "assistant", + # # "name": None, + # "tool_call_id": None, + # "content": "?", + # "tool_calls": [ + # { + # # "id": "call_531873", + # "type": "function", + # "function": { + # "name": "add", + # "arguments": { + # "a": 2535, + # "b": 32222000403 + # } + # } + # } + # ] + # }), + # Message(**{ + # "role": "tool", + # "name": "add", + # "tool_call_id": "call_531873", + # "content": "32222002938", + # "tool_calls": None + # }) + # ]) diff --git a/examples/openai/prompting.py b/examples/openai/prompting.py index 1ec2a11e3..d2a0cd129 100644 --- a/examples/openai/prompting.py +++ b/examples/openai/prompting.py @@ -53,14 +53,13 @@ def raise_exception(msg: str): class ChatTemplate(BaseModel): template: str inferred_tool_style: Optional['ToolsPromptStyle'] = None + eos_token: str + bos_token: str def __init__(self, template: str, eos_token: str, bos_token: str): - super().__init__(template=template - ) + super().__init__(template=template, eos_token=eos_token, bos_token=bos_token) env = jinja2.Environment(loader=jinja2.BaseLoader(), trim_blocks=True, lstrip_blocks=True) self._template = env.from_string(template) - self._eos_token = eos_token - self._bos_token = bos_token self._strict_user_assistant_alternation = "{% if (message['role'] == 'user') != (loop.index0 % 2 == 0) %}{{ raise_exception" in template @@ -93,9 +92,6 @@ class ChatTemplate(BaseModel): sys.stderr.write(f"Expected suffix ({self._suffix}) not found: {s}\n") return s - def __str__(self): - return f"ChatTemplate(template={self.template}, eos_token={self._eos_token}, bos_token={self._bos_token})" - def add_system_prompt(self, messages: list[Message], system_prompt: Message) -> list[Message]: assert system_prompt.role == "system" # TODO: add to last system message, or create a new one just before the last user message @@ -194,8 +190,8 @@ class ChatTemplate(BaseModel): result = self._template.render( messages=messages, - eos_token=self._eos_token, - bos_token='' if omit_bos else self._bos_token, + eos_token=self.eos_token, + bos_token='' if omit_bos else self.bos_token, raise_exception=raise_exception, add_generation_prompt=add_generation_prompt, ) @@ -339,7 +335,7 @@ class Hermes2ProToolsChatHandler(ToolCallTagsChatHandler): except ImportError: raise ImportError(f"Please `git clone https://github.com/NousResearch/Hermes-Function-Calling {path}`") - prompt = PromptManager().generate_prompt(user_prompt=[], tools=[json.dumps(tool) for tool in args.tools]) + prompt = PromptManager().generate_prompt(user_prompt=[], tools=[tool.model_dump_json() for tool in args.tools]) assert len(prompt) == 1 and prompt[0]["role"] == "system" self.output_format_prompt = Message(**prompt[0]) @@ -347,9 +343,6 @@ class FunctionaryToolsChatHandler(ChatHandler): def __init__(self, args: ChatHandlerArgs, parallel_calls: bool): super().__init__(args) - # Only allowing a single tool call at a time for now. - # Note that if there were more, they'd be separated by a '<|from|>assistant' literal - self.output_format_prompt = Message( role="system", content= '// Supported function definitions that should be called when necessary.\n' + @@ -585,19 +578,19 @@ def get_chat_handler(args: ChatHandlerArgs, parallel_calls: bool, tool_style: Op return TemplatedToolsChatHandler(args, _LONG_TEMPLATE, parallel_calls=parallel_calls, escapes_underscores=True) elif tool_style == ToolsPromptStyle.TOOLS_HERMES_2_PRO: - return Hermes2ProToolsChatHandler(args) + return Hermes2ProToolsChatHandler(args, parallel_calls=parallel_calls) else: raise ValueError(f"Unsupported tool call style: {args.chat_template.tool_style}") _ts_converter = SchemaToTypeScriptConverter() def _please_respond_with_schema(schema: dict) -> str: - # sig = json.dumps(schema, indent=2) - sig = _ts_converter.visit(schema) + sig = json.dumps(schema, indent=2) + # sig = _ts_converter.visit(schema) return f'Please respond in JSON format with the following schema: {sig}' def _tools_typescript_signatures(tools: list[Tool]) -> str: - return 'namespace functions {' + '\n'.join( + return 'namespace functions {\n' + '\n'.join( '// ' + tool.function.description.replace('\n', '\n// ') + '\n' + '' 'type ' + tool.function.name + ' = (_: ' + _ts_converter.visit(tool.function.parameters) + ") => any;\n" for tool in tools diff --git a/examples/openai/ts_converter.py b/examples/openai/ts_converter.py index 108c1482e..7ba5c439f 100644 --- a/examples/openai/ts_converter.py +++ b/examples/openai/ts_converter.py @@ -24,13 +24,13 @@ class SchemaToTypeScriptConverter: elif additional_properties == False: additional_properties = None - return "{" + ', '.join([ + return "{\n" + ',\n'.join([ f'{self._desc_comment(prop_schema)}{prop_name}{"" if prop_name in required else "?"}: {self.visit(prop_schema)}' for prop_name, prop_schema in properties ] + ( [f"{self._desc_comment(additional_properties) if additional_properties else ''}[key: string]: {self.visit(additional_properties)}"] if additional_properties is not None else [] - )) + "}" + )) + "\n}" def visit(self, schema: dict): def print_constant(v):