diff --git a/examples/openai/prompting.py b/examples/openai/prompting.py index f274df6a8..222e133f0 100644 --- a/examples/openai/prompting.py +++ b/examples/openai/prompting.py @@ -88,12 +88,9 @@ class ChatTemplate(BaseModel): def probe_template_capabilities(self): - def test(messages: list[Message]): - return self._template.render(messages=messages, eos_token=self.eos_token, bos_token=self.bos_token, raise_exception=raise_exception, add_generation_prompt=True) - def succeeds(messages: list[Message], strings_to_find = ()): try: - result = test(messages) + result = self.raw_render(messages, add_generation_prompt=True) # print(result) for s in strings_to_find: if s not in result: @@ -133,8 +130,8 @@ class ChatTemplate(BaseModel): delimiter = '<%$[SAMPLE]$%>' user_msg = Message(role="user", content="Hey") - empty_prompt = self.render([user_msg], add_generation_prompt=True).strip() - planted_prompt = self.render([user_msg, Message(role="assistant", content=delimiter)], add_generation_prompt=False).strip() + empty_prompt = self.raw_render([user_msg], add_generation_prompt=True).strip() + planted_prompt = self.raw_render([user_msg, Message(role="assistant", content=delimiter)], add_generation_prompt=False).strip() assert planted_prompt.startswith(empty_prompt), f"Planted prompt does not start with empty prompt: {planted_prompt} vs {empty_prompt}" [prefix, suffix] = planted_prompt[len(empty_prompt):].split(delimiter) @@ -181,10 +178,59 @@ class ChatTemplate(BaseModel): bos_token = tokenizer.bos_token, eos_token = tokenizer.eos_token) - def render(self, messages: list[Message], add_generation_prompt: bool, omit_bos: bool = False): + def raw_render(self, messages: list[Message], add_generation_prompt: bool, omit_bos: bool = False): + result = self._template.render( + messages=messages, + eos_token=self.eos_token, + bos_token='' if omit_bos else self.bos_token, + raise_exception=raise_exception, + add_generation_prompt=add_generation_prompt, + ) + return result + +class ChatHandlerArgs(BaseModel): + chat_template: ChatTemplate + response_schema: Optional[dict] = None + tools: Optional[list[Tool]] = None + +class ChatHandler(ABC): + def __init__(self, args: ChatHandlerArgs, style: Optional[ToolsPromptStyle]): + self.args = args + self.style = style + self.output_format_prompt: Optional[Message] = None + self.grammar: Optional[str] = None + + @abstractmethod + def parse(self, s: str) -> Optional[Message]: + raise NotImplementedError() + + def render_prompt(self, messages: list[Message]) -> str: def normalize(m: Message): + if self.style == ToolsPromptStyle.TOOLS_THOUGHTFUL_STEPS: + if m.tool_calls: + m = Message( + role=m.role, + content=json.dumps({ + _THOUGHT_KEY: m.content or '', + "next_step": { + "tool_calls": [tc.model_dump() for tc in m.tool_calls] + } + }, indent=2) + ) + else: + m = Message( + role=m.role, + content=json.dumps({ + _THOUGHT_KEY: '', + "next_step": { + "result": m.content + } + }, indent=2) + ) + # Fall through to benefit from role normalization + if m.tool_calls: - if not self.formats_tool_call or not self.formats_tool_call_content: + if not self.args.chat_template.formats_tool_call or not self.args.chat_template.formats_tool_call_content: return Message( role=m.role, content='\n'.join([ @@ -195,7 +241,7 @@ class ChatTemplate(BaseModel): ]) ]) ) - elif self.expects_stringified_function_arguments: + elif self.args.chat_template.expects_stringified_function_arguments: return Message( role=m.role, content=m.content, @@ -215,7 +261,7 @@ class ChatTemplate(BaseModel): ) else: return m - elif self.expects_strict_user_assistant_alternance and m.role not in ('user', 'assistant'): + elif self.args.chat_template.expects_strict_user_assistant_alternance and m.role not in ('user', 'assistant'): if m.role == "system": return Message(role="user", content=f'[SYS]{m.content}[/SYS]') elif m.role == "tool": @@ -228,7 +274,7 @@ class ChatTemplate(BaseModel): messages=[normalize(m) for m in messages] - if self.expects_strict_user_assistant_alternance: + if self.args.chat_template.expects_strict_user_assistant_alternance: new_messages=[] current_role = 'user' current_content = [] @@ -237,7 +283,7 @@ class ChatTemplate(BaseModel): nonlocal current_content nonlocal current_role - if self.expects_strict_user_assistant_alternance or current_content: + if self.args.chat_template.expects_strict_user_assistant_alternance or current_content: new_messages.append(Message( role=current_role, content='\n'.join(current_content) @@ -263,7 +309,7 @@ class ChatTemplate(BaseModel): messages = [m.model_dump() for m in messages] # if self.inferred_tool_style == ToolsPromptStyle.TYPESCRIPT_FUNCTIONARY_V2: - if self.expects_stringified_function_arguments: + if self.args.chat_template.expects_stringified_function_arguments: messages = [ { **m, @@ -281,33 +327,14 @@ class ChatTemplate(BaseModel): for m in messages ] - result = self._template.render( + return self.args.chat_template.raw_render( messages=messages, - eos_token=self.eos_token, - bos_token='' if omit_bos else self.bos_token, - raise_exception=raise_exception, - add_generation_prompt=add_generation_prompt, + add_generation_prompt=True, ) - return result - -class ChatHandlerArgs(BaseModel): - chat_template: ChatTemplate - response_schema: Optional[dict] = None - tools: Optional[list[Tool]] = None - -class ChatHandler(ABC): - def __init__(self, args: ChatHandlerArgs): - self.args = args - self.output_format_prompt: Optional[Message] = None - self.grammar: Optional[str] = None - - @abstractmethod - def parse(self, s: str) -> Optional[Message]: - raise NotImplementedError() class NoToolsChatHandler(ChatHandler): def __init__(self, args: ChatHandlerArgs): - super().__init__(args) + super().__init__(args, None) assert not args.tools if args.response_schema: @@ -327,8 +354,8 @@ class NoToolsChatHandler(ChatHandler): return Message(role="assistant", content=s) class ToolCallTagsChatHandler(ChatHandler): - def __init__(self, args: ChatHandlerArgs, escapes_underscores: bool, parallel_calls: bool): - super().__init__(args) + def __init__(self, args: ChatHandlerArgs, style: Optional[ToolsPromptStyle], escapes_underscores: bool, parallel_calls: bool): + super().__init__(args, style) converter = SchemaConverter(prop_order={}, allow_fetch=False, dotall=False, raw_pattern=False) tool_rules = [] @@ -404,8 +431,8 @@ class ToolCallTagsChatHandler(ChatHandler): class TemplatedToolsChatHandler(ToolCallTagsChatHandler): - def __init__(self, args: ChatHandlerArgs, template: str, parallel_calls: bool, escapes_underscores: bool = False): - super().__init__(args, escapes_underscores=escapes_underscores, parallel_calls=parallel_calls) + def __init__(self, args: ChatHandlerArgs, template: str, parallel_calls: bool, escapes_underscores: bool = False, style: Optional[ToolsPromptStyle] = None): + super().__init__(args, style=style, escapes_underscores=escapes_underscores, parallel_calls=parallel_calls) assert '{tools}' in template, 'Template must contain "{tools}"' self.output_format_prompt = Message( @@ -418,7 +445,7 @@ class TemplatedToolsChatHandler(ToolCallTagsChatHandler): class Hermes2ProToolsChatHandler(ToolCallTagsChatHandler): def __init__(self, args: ChatHandlerArgs, parallel_calls: bool): - super().__init__(args, escapes_underscores=False, parallel_calls=parallel_calls) + super().__init__(args, style=ToolsPromptStyle.TOOLS_HERMES_2_PRO, escapes_underscores=False, parallel_calls=parallel_calls) # Hackily import https://github.com/NousResearch/Hermes-Function-Calling path = str(Path(__file__).parent / "hermes_function_calling") @@ -434,7 +461,7 @@ class Hermes2ProToolsChatHandler(ToolCallTagsChatHandler): class FunctionaryToolsChatHandler(ChatHandler): def __init__(self, args: ChatHandlerArgs, parallel_calls: bool): - super().__init__(args) + super().__init__(args, ToolsPromptStyle.TYPESCRIPT_FUNCTIONARY_V2) self.output_format_prompt = Message( role="system", @@ -541,9 +568,9 @@ def _make_bespoke_schema(response_schema, tool_call_schema, parallel_calls): # "required": ["next_step"] } -class BespokeToolsChatHandler(ChatHandler): +class ThoughtfulStepsToolsChatHandler(ChatHandler): def __init__(self, args: ChatHandlerArgs, parallel_calls: bool): - super().__init__(args) + super().__init__(args, ToolsPromptStyle.TOOLS_THOUGHTFUL_STEPS) # args.response_schema = args.response_schema or {} converter = SchemaConverter(prop_order={}, allow_fetch=False, dotall=False, raw_pattern=False) @@ -660,7 +687,7 @@ def get_chat_handler(args: ChatHandlerArgs, parallel_calls: bool, tool_style: Op return NoToolsChatHandler(args) elif tool_style == ToolsPromptStyle.TOOLS_THOUGHTFUL_STEPS: - return BespokeToolsChatHandler(args, parallel_calls=parallel_calls) + return ThoughtfulStepsToolsChatHandler(args, parallel_calls=parallel_calls) elif tool_style == ToolsPromptStyle.TYPESCRIPT_FUNCTIONARY_V2: return FunctionaryToolsChatHandler(args, parallel_calls=parallel_calls) diff --git a/examples/openai/server.py b/examples/openai/server.py index 6d19f12f1..474f07489 100644 --- a/examples/openai/server.py +++ b/examples/openai/server.py @@ -140,7 +140,7 @@ def main( if chat_handler.output_format_prompt: messages = chat_template.add_system_prompt(messages, chat_handler.output_format_prompt) - prompt = chat_template.render(messages, add_generation_prompt=True) + prompt = chat_handler.render_prompt(messages) if verbose: sys.stderr.write(f'\n# REQUEST:\n\n{chat_request.model_dump_json(indent=2)}\n\n') diff --git a/examples/openai/test_chat_handlers.md b/examples/openai/test_chat_handlers.md index 6f80964e9..f5fc81c44 100644 --- a/examples/openai/test_chat_handlers.md +++ b/examples/openai/test_chat_handlers.md @@ -47,23 +47,6 @@ Messages: ``` -# mistral_instruct_v0_1 - - -Template: - -```js -{{ bos_token }}{% for message in messages %}{% if (message['role'] == 'user') != (loop.index0 % 2 == 0) %}{{ raise_exception('Conversation roles must alternate user/assistant/user/assistant/...') }}{% endif %}{% if message['role'] == 'user' %}{{ '[INST] ' + message['content'] + ' [/INST]' }}{% elif message['role'] == 'assistant' %}{{ message['content'] + eos_token}}{% else %}{{ raise_exception('Only user and assistant roles are supported!') }}{% endif %}{% endfor %} -``` - - -Prompt: - -```js -[INST] Add two numbers for the purpose of this test. [/INST]{"id": "call_531873", "type": "function", "function": {"name": "superSecretTool", "arguments": {"a": 2535, "b": 32222000403}}}[INST] [TOOL(name=superSecretTool, id=call_531873)]32222002938[/TOOL] [/INST]The sum of 2535 and 32222000403 is 42. -``` - - ## ToolsPromptStyle.TOOLS_THOUGHTFUL_STEPS @@ -211,6 +194,55 @@ thought-about-next-step-only-kv ::= "\"thought_about_next_step_only\"" space ":" ``` +# mistral_instruct_v0_1 + + +Template: + +```js +{{ bos_token }}{% for message in messages %}{% if (message['role'] == 'user') != (loop.index0 % 2 == 0) %}{{ raise_exception('Conversation roles must alternate user/assistant/user/assistant/...') }}{% endif %}{% if message['role'] == 'user' %}{{ '[INST] ' + message['content'] + ' [/INST]' }}{% elif message['role'] == 'assistant' %}{{ message['content'] + eos_token}}{% else %}{{ raise_exception('Only user and assistant roles are supported!') }}{% endif %}{% endfor %} +``` + + +Prompt: + +```js +[INST] { + "thought_about_next_step_only": "", + "next_step": { + "result": "Add two numbers for the purpose of this test." + } +} [/INST]{ + "thought_about_next_step_only": "", + "next_step": { + "tool_calls": [ + { + "id": "call_531873", + "type": "function", + "function": { + "name": "superSecretTool", + "arguments": { + "a": 2535, + "b": 32222000403 + } + } + } + ] + } +}[INST] [TOOL(name=None, id=None)]{ + "thought_about_next_step_only": "", + "next_step": { + "result": "32222002938" + } +}[/TOOL] [/INST]{ + "thought_about_next_step_only": "", + "next_step": { + "result": "The sum of 2535 and 32222000403 is 42." + } +} +``` + + ### without tools @@ -233,6 +265,23 @@ space ::= " "? ``` +# mistral_instruct_v0_1 + + +Template: + +```js +{{ bos_token }}{% for message in messages %}{% if (message['role'] == 'user') != (loop.index0 % 2 == 0) %}{{ raise_exception('Conversation roles must alternate user/assistant/user/assistant/...') }}{% endif %}{% if message['role'] == 'user' %}{{ '[INST] ' + message['content'] + ' [/INST]' }}{% elif message['role'] == 'assistant' %}{{ message['content'] + eos_token}}{% else %}{{ raise_exception('Only user and assistant roles are supported!') }}{% endif %}{% endfor %} +``` + + +Prompt: + +```js +[INST] Add two numbers for the purpose of this test. [/INST]{"id": "call_531873", "type": "function", "function": {"name": "superSecretTool", "arguments": {"a": 2535, "b": 32222000403}}}[INST] [TOOL(name=superSecretTool, id=call_531873)]32222002938[/TOOL] [/INST]The sum of 2535 and 32222000403 is 42. +``` + + ## ToolsPromptStyle.TOOLS_MIXTRAL @@ -322,6 +371,178 @@ tool-call ::= "" space (superSecretTool-tool-call | say-t ``` +# mistral_instruct_v0_1 + + +Template: + +```js +{{ bos_token }}{% for message in messages %}{% if (message['role'] == 'user') != (loop.index0 % 2 == 0) %}{{ raise_exception('Conversation roles must alternate user/assistant/user/assistant/...') }}{% endif %}{% if message['role'] == 'user' %}{{ '[INST] ' + message['content'] + ' [/INST]' }}{% elif message['role'] == 'assistant' %}{{ message['content'] + eos_token}}{% else %}{{ raise_exception('Only user and assistant roles are supported!') }}{% endif %}{% endfor %} +``` + + +Prompt: + +```js +[INST] Add two numbers for the purpose of this test. [/INST]{"id": "call_531873", "type": "function", "function": {"name": "superSecretTool", "arguments": {"a": 2535, "b": 32222000403}}}[INST] [TOOL(name=superSecretTool, id=call_531873)]32222002938[/TOOL] [/INST]The sum of 2535 and 32222000403 is 42. +``` + + +### without tools + + +Prompt: + +```json +Please respond in JSON format with the following schema: { + "type": "integer" +} +``` + + +Grammar: + +```js +decimal-part ::= [0-9] [0-9]? [0-9]? [0-9]? [0-9]? [0-9]? [0-9]? [0-9]? [0-9]? [0-9]? [0-9]? +integral-part ::= [0-9] | [1-9] [0-9]? [0-9]? [0-9]? [0-9]? [0-9]? [0-9]? [0-9]? [0-9]? [0-9]? [0-9]? +root ::= ("-"? integral-part) space +space ::= " "? +``` + + +# mistral_instruct_v0_1 + + +Template: + +```js +{{ bos_token }}{% for message in messages %}{% if (message['role'] == 'user') != (loop.index0 % 2 == 0) %}{{ raise_exception('Conversation roles must alternate user/assistant/user/assistant/...') }}{% endif %}{% if message['role'] == 'user' %}{{ '[INST] ' + message['content'] + ' [/INST]' }}{% elif message['role'] == 'assistant' %}{{ message['content'] + eos_token}}{% else %}{{ raise_exception('Only user and assistant roles are supported!') }}{% endif %}{% endfor %} +``` + + +Prompt: + +```js +[INST] Add two numbers for the purpose of this test. [/INST]{"id": "call_531873", "type": "function", "function": {"name": "superSecretTool", "arguments": {"a": 2535, "b": 32222000403}}}[INST] [TOOL(name=superSecretTool, id=call_531873)]32222002938[/TOOL] [/INST]The sum of 2535 and 32222000403 is 42. +``` + + +## ToolsPromptStyle.TYPESCRIPT_FUNCTIONARY_V2 + + +### with tools + + +Prompt: + +```json +// Supported function definitions that should be called when necessary. +namespace functions { +// Adds two numbers +type superSecretTool = (_: { +a: number, +b: number +}) => any; + +// Says something out loud (TTS) +type say = (_: { +// The text to say out loud +text: string +}) => any; +} // namespace functions +``` + + +Grammar: + +```js +content ::= start content-without-start +content-without-start ::= "all\n<|content|>" not-from* +decimal-part ::= [0-9] [0-9]? [0-9]? [0-9]? [0-9]? [0-9]? [0-9]? [0-9]? [0-9]? [0-9]? [0-9]? +integer ::= ("-"? integral-part) space +integral-part ::= [0-9] | [1-9] [0-9]? [0-9]? [0-9]? [0-9]? [0-9]? [0-9]? [0-9]? [0-9]? [0-9]? [0-9]? +not-from ::= ([^<] | "<" ([^|] | "|" ([^f] | "f" ([^r] | "r" ([^o] | "o" ([^m] | "m" ([^|] | "|" ([^>])?)?)?)?)?)?)?) +root ::= content-without-start content* (tool-call+ content*)? | tool-call-without-start tool-call* content* +say-args ::= "{" space say-args-text-kv "}" space +say-args-text-kv ::= "\"text\"" space ":" space string +say-call ::= "say" "\n<|content|>\n" say-args "\n" +space ::= " "? +start ::= "<|from|>assistant\n<|recipient|>" +string ::= "\"" ( + [^"\\] | + "\\" (["\\/bfnrt] | "u" [0-9a-fA-F] [0-9a-fA-F] [0-9a-fA-F] [0-9a-fA-F]) + )* "\"" space +superSecretTool-args ::= "{" space superSecretTool-args-a-kv "," space superSecretTool-args-b-kv "}" space +superSecretTool-args-a-kv ::= "\"a\"" space ":" space integer +superSecretTool-args-b-kv ::= "\"b\"" space ":" space integer +superSecretTool-call ::= "superSecretTool" "\n<|content|>\n" superSecretTool-args "\n" +tool-call ::= start tool-call-without-start +tool-call-without-start ::= superSecretTool-call | say-call +``` + + +# functionary_v2_2 + + +Template: + +```js +{#v2.2#} +{% for message in messages %} +{% if message['role'] == 'user' or message['role'] == 'system' %} +{{ '<|from|>' + message['role'] + ' +<|recipient|>all +<|content|>' + message['content'] + ' +' }}{% elif message['role'] == 'tool' %} +{{ '<|from|>' + message['name'] + ' +<|recipient|>all +<|content|>' + message['content'] + ' +' }}{% else %} +{% set contain_content='no'%} +{% if message['content'] is not none %} +{{ '<|from|>assistant +<|recipient|>all +<|content|>' + message['content'] }}{% set contain_content='yes'%} +{% endif %} +{% if 'tool_calls' in message and message['tool_calls'] is not none %} +{% for tool_call in message['tool_calls'] %} +{% set prompt='<|from|>assistant +<|recipient|>' + tool_call['function']['name'] + ' +<|content|>' + tool_call['function']['arguments'] %} +{% if loop.index == 1 and contain_content == "no" %} +{{ prompt }}{% else %} +{{ ' +' + prompt}}{% endif %} +{% endfor %} +{% endif %} +{{ '<|stop|> +' }}{% endif %} +{% endfor %} +{% if add_generation_prompt %}{{ '<|from|>assistant +<|recipient|>' }}{% endif %} +``` + + +Prompt: + +```js +<|from|>user +<|recipient|>all +<|content|>Add two numbers for the purpose of this test. +<|from|>assistant +<|recipient|>all +<|content|>{"id": "call_531873", "type": "function", "function": {"name": "superSecretTool", "arguments": {"a": 2535, "b": 32222000403}}}<|stop|> +<|from|>superSecretTool +<|recipient|>all +<|content|>32222002938 +<|from|>assistant +<|recipient|>all +<|content|>The sum of 2535 and 32222000403 is 42.<|stop|> +<|from|>assistant +<|recipient|> +``` + + ### without tools @@ -406,7 +627,7 @@ Prompt: ``` -## ToolsPromptStyle.TYPESCRIPT_FUNCTIONARY_V2 +## ToolsPromptStyle.TOOLS_SHORT ### with tools @@ -415,48 +636,667 @@ Prompt: Prompt: ```json -// Supported function definitions that should be called when necessary. -namespace functions { -// Adds two numbers -type superSecretTool = (_: { -a: number, -b: number -}) => any; - -// Says something out loud (TTS) -type say = (_: { -// The text to say out loud -text: string -}) => any; -} // namespace functions +Here are the tools available: + +{ + "type": "function", + "function": { + "name": "superSecretTool", + "description": "Adds two numbers", + "parameters": { + "properties": { + "a": { + "type": "integer" + }, + "b": { + "type": "integer" + } + }, + "required": [ + "a", + "b" + ] + } + } +} +{ + "type": "function", + "function": { + "name": "say", + "description": "Says something out loud (TTS)", + "parameters": { + "properties": { + "text": { + "description": "The text to say out loud", + "type": "string" + } + }, + "required": [ + "text" + ] + } + } +} + ``` Grammar: ```js -content ::= start content-without-start -content-without-start ::= "all\n<|content|>" not-from* +content ::= [^<] | "<" [^t<] | "])?)?)?)?)?)?)?) -root ::= content-without-start content* (tool-call+ content*)? | tool-call-without-start tool-call* content* -say-args ::= "{" space say-args-text-kv "}" space -say-args-text-kv ::= "\"text\"" space ":" space string -say-call ::= "say" "\n<|content|>\n" say-args "\n" +root ::= content* tool-call? +say-tool-call ::= "{" space say-tool-call-name-kv "," space say-tool-call-arguments-kv "}" space +say-tool-call-arguments ::= "{" space say-tool-call-arguments-text-kv "}" space +say-tool-call-arguments-kv ::= "\"arguments\"" space ":" space say-tool-call-arguments +say-tool-call-arguments-text-kv ::= "\"text\"" space ":" space string +say-tool-call-name ::= "\"say\"" +say-tool-call-name-kv ::= "\"name\"" space ":" space say-tool-call-name space ::= " "? -start ::= "<|from|>assistant\n<|recipient|>" string ::= "\"" ( [^"\\] | "\\" (["\\/bfnrt] | "u" [0-9a-fA-F] [0-9a-fA-F] [0-9a-fA-F] [0-9a-fA-F]) )* "\"" space -superSecretTool-args ::= "{" space superSecretTool-args-a-kv "," space superSecretTool-args-b-kv "}" space -superSecretTool-args-a-kv ::= "\"a\"" space ":" space integer -superSecretTool-args-b-kv ::= "\"b\"" space ":" space integer -superSecretTool-call ::= "superSecretTool" "\n<|content|>\n" superSecretTool-args "\n" -tool-call ::= start tool-call-without-start -tool-call-without-start ::= superSecretTool-call | say-call +superSecretTool-tool-call ::= "{" space superSecretTool-tool-call-name-kv "," space superSecretTool-tool-call-arguments-kv "}" space +superSecretTool-tool-call-arguments ::= "{" space superSecretTool-tool-call-arguments-a-kv "," space superSecretTool-tool-call-arguments-b-kv "}" space +superSecretTool-tool-call-arguments-a-kv ::= "\"a\"" space ":" space integer +superSecretTool-tool-call-arguments-b-kv ::= "\"b\"" space ":" space integer +superSecretTool-tool-call-arguments-kv ::= "\"arguments\"" space ":" space superSecretTool-tool-call-arguments +superSecretTool-tool-call-name ::= "\"superSecretTool\"" +superSecretTool-tool-call-name-kv ::= "\"name\"" space ":" space superSecretTool-tool-call-name +tool-call ::= "" space (superSecretTool-tool-call | say-tool-call) space "" +``` + + +# hermes_2_pro_mistral + + +Template: + +```js +{% for message in messages %}{{'<|im_start|>' + message['role'] + ' +' + message['content'] + '<|im_end|>' + ' +'}}{% endfor %}{% if add_generation_prompt %}{{ '<|im_start|>assistant +' }}{% endif %} +``` + + +Prompt: + +```js +<|im_start|>user +Add two numbers for the purpose of this test.<|im_end|> +<|im_start|>assistant +{"id": "call_531873", "type": "function", "function": {"name": "superSecretTool", "arguments": {"a": 2535, "b": 32222000403}}}<|im_end|> +<|im_start|>tool +32222002938<|im_end|> +<|im_start|>assistant +The sum of 2535 and 32222000403 is 42.<|im_end|> +<|im_start|>assistant + +``` + + +### without tools + + +Prompt: + +```json +Please respond in JSON format with the following schema: { + "type": "integer" +} +``` + + +Grammar: + +```js +decimal-part ::= [0-9] [0-9]? [0-9]? [0-9]? [0-9]? [0-9]? [0-9]? [0-9]? [0-9]? [0-9]? [0-9]? +integral-part ::= [0-9] | [1-9] [0-9]? [0-9]? [0-9]? [0-9]? [0-9]? [0-9]? [0-9]? [0-9]? [0-9]? [0-9]? +root ::= ("-"? integral-part) space +space ::= " "? +``` + + +# hermes_2_pro_mistral + + +Template: + +```js +{% for message in messages %}{{'<|im_start|>' + message['role'] + ' +' + message['content'] + '<|im_end|>' + ' +'}}{% endfor %}{% if add_generation_prompt %}{{ '<|im_start|>assistant +' }}{% endif %} +``` + + +Prompt: + +```js +<|im_start|>user +Add two numbers for the purpose of this test.<|im_end|> +<|im_start|>assistant +{"id": "call_531873", "type": "function", "function": {"name": "superSecretTool", "arguments": {"a": 2535, "b": 32222000403}}}<|im_end|> +<|im_start|>tool +32222002938<|im_end|> +<|im_start|>assistant +The sum of 2535 and 32222000403 is 42.<|im_end|> +<|im_start|>assistant + +``` + + +## ToolsPromptStyle.TOOLS_LONG + + +### with tools + + +Prompt: + +```json +Call one or more functions to assist with the user query, every time this is possible. Don't make assumptions about what values to plug into functions. Here are the available tools: + +{ + "type": "function", + "function": { + "name": "superSecretTool", + "description": "Adds two numbers", + "parameters": { + "properties": { + "a": { + "type": "integer" + }, + "b": { + "type": "integer" + } + }, + "required": [ + "a", + "b" + ] + } + } +} +{ + "type": "function", + "function": { + "name": "say", + "description": "Says something out loud (TTS)", + "parameters": { + "properties": { + "text": { + "description": "The text to say out loud", + "type": "string" + } + }, + "required": [ + "text" + ] + } + } +} + + +To call each function, give its name and arguments within XML tags as follows: + +{"name": , "arguments": } + +``` + + +Grammar: + +```js +content ::= [^<] | "<" [^t<] | "" +``` + + +# hermes_2_pro_mistral + + +Template: + +```js +{% for message in messages %}{{'<|im_start|>' + message['role'] + ' +' + message['content'] + '<|im_end|>' + ' +'}}{% endfor %}{% if add_generation_prompt %}{{ '<|im_start|>assistant +' }}{% endif %} +``` + + +Prompt: + +```js +<|im_start|>user +Add two numbers for the purpose of this test.<|im_end|> +<|im_start|>assistant +{"id": "call_531873", "type": "function", "function": {"name": "superSecretTool", "arguments": {"a": 2535, "b": 32222000403}}}<|im_end|> +<|im_start|>tool +32222002938<|im_end|> +<|im_start|>assistant +The sum of 2535 and 32222000403 is 42.<|im_end|> +<|im_start|>assistant + +``` + + +### without tools + + +Prompt: + +```json +Please respond in JSON format with the following schema: { + "type": "integer" +} +``` + + +Grammar: + +```js +decimal-part ::= [0-9] [0-9]? [0-9]? [0-9]? [0-9]? [0-9]? [0-9]? [0-9]? [0-9]? [0-9]? [0-9]? +integral-part ::= [0-9] | [1-9] [0-9]? [0-9]? [0-9]? [0-9]? [0-9]? [0-9]? [0-9]? [0-9]? [0-9]? [0-9]? +root ::= ("-"? integral-part) space +space ::= " "? +``` + + +# hermes_2_pro_mistral + + +Template: + +```js +{% for message in messages %}{{'<|im_start|>' + message['role'] + ' +' + message['content'] + '<|im_end|>' + ' +'}}{% endfor %}{% if add_generation_prompt %}{{ '<|im_start|>assistant +' }}{% endif %} +``` + + +Prompt: + +```js +<|im_start|>user +Add two numbers for the purpose of this test.<|im_end|> +<|im_start|>assistant +{"id": "call_531873", "type": "function", "function": {"name": "superSecretTool", "arguments": {"a": 2535, "b": 32222000403}}}<|im_end|> +<|im_start|>tool +32222002938<|im_end|> +<|im_start|>assistant +The sum of 2535 and 32222000403 is 42.<|im_end|> +<|im_start|>assistant + +``` + + +## ToolsPromptStyle.TOOLS_THOUGHTFUL_STEPS + + +### with tools + + +Prompt: + +```json +You are a function calling AI model. +Here are the tools available: +{ + "type": "function", + "function": { + "name": "superSecretTool", + "description": "Adds two numbers", + "parameters": { + "properties": { + "a": { + "type": "integer" + }, + "b": { + "type": "integer" + } + }, + "required": [ + "a", + "b" + ] + } + } +} +{ + "type": "function", + "function": { + "name": "say", + "description": "Says something out loud (TTS)", + "parameters": { + "properties": { + "text": { + "description": "The text to say out loud", + "type": "string" + } + }, + "required": [ + "text" + ] + } + } +} +Please respond in JSON format with the following schema: { + "type": "object", + "properties": { + "thought_about_next_step_only": { + "title": "Thought about next step", + "type": "string" + }, + "next_step": { + "title": "Next Step: either a result or one or more tool calls to achieve the original goal", + "oneOf": [ + { + "properties": { + "tool_calls": { + "prefixItems": [ + { + "properties": { + "name": { + "title": "Name of the tool to call", + "type": "string" + }, + "arguments": { + "title": "Arguments to pass to the tool", + "type": "object" + } + }, + "required": [ + "name", + "arguments" + ] + } + ] + } + }, + "required": [ + "tool_calls" + ] + }, + { + "title": "Result (achieving original goal)", + "properties": { + "result": { + "type": "integer" + } + }, + "required": [ + "result" + ] + } + ] + } + }, + "required": [ + "original_goal", + "thought_about_next_step_only", + "next_step" + ] +} +``` + + +Grammar: + +```js +decimal-part ::= [0-9] [0-9]? [0-9]? [0-9]? [0-9]? [0-9]? [0-9]? [0-9]? [0-9]? [0-9]? [0-9]? +integer ::= ("-"? integral-part) space +integral-part ::= [0-9] | [1-9] [0-9]? [0-9]? [0-9]? [0-9]? [0-9]? [0-9]? [0-9]? [0-9]? [0-9]? [0-9]? +next-step ::= next-step-0 | next-step-1 +next-step-0 ::= "{" space next-step-0-tool-calls-kv "}" space +next-step-0-tool-calls ::= "[" space next-step-0-tool-calls-tuple-0 "]" space +next-step-0-tool-calls-kv ::= "\"tool_calls\"" space ":" space next-step-0-tool-calls +next-step-0-tool-calls-tuple-0 ::= next-step-0-tool-calls-tuple-0-0 | next-step-0-tool-calls-tuple-0-1 +next-step-0-tool-calls-tuple-0-0 ::= "{" space next-step-0-tool-calls-tuple-0-0-name-kv "," space next-step-0-tool-calls-tuple-0-0-arguments-kv "}" space +next-step-0-tool-calls-tuple-0-0-arguments ::= "{" space next-step-0-tool-calls-tuple-0-0-arguments-a-kv "," space next-step-0-tool-calls-tuple-0-0-arguments-b-kv "}" space +next-step-0-tool-calls-tuple-0-0-arguments-a-kv ::= "\"a\"" space ":" space integer +next-step-0-tool-calls-tuple-0-0-arguments-b-kv ::= "\"b\"" space ":" space integer +next-step-0-tool-calls-tuple-0-0-arguments-kv ::= "\"arguments\"" space ":" space next-step-0-tool-calls-tuple-0-0-arguments +next-step-0-tool-calls-tuple-0-0-name ::= "\"superSecretTool\"" +next-step-0-tool-calls-tuple-0-0-name-kv ::= "\"name\"" space ":" space next-step-0-tool-calls-tuple-0-0-name +next-step-0-tool-calls-tuple-0-1 ::= "{" space next-step-0-tool-calls-tuple-0-1-name-kv "," space next-step-0-tool-calls-tuple-0-1-arguments-kv "}" space +next-step-0-tool-calls-tuple-0-1-arguments ::= "{" space next-step-0-tool-calls-tuple-0-1-arguments-text-kv "}" space +next-step-0-tool-calls-tuple-0-1-arguments-kv ::= "\"arguments\"" space ":" space next-step-0-tool-calls-tuple-0-1-arguments +next-step-0-tool-calls-tuple-0-1-arguments-text-kv ::= "\"text\"" space ":" space string +next-step-0-tool-calls-tuple-0-1-name ::= "\"say\"" +next-step-0-tool-calls-tuple-0-1-name-kv ::= "\"name\"" space ":" space next-step-0-tool-calls-tuple-0-1-name +next-step-1 ::= "{" space next-step-1-result-kv "}" space +next-step-1-result-kv ::= "\"result\"" space ":" space integer +next-step-kv ::= "\"next_step\"" space ":" space next-step +root ::= "{" space thought-about-next-step-only-kv "," space next-step-kv "}" space +space ::= " "? +string ::= "\"" ( + [^"\\] | + "\\" (["\\/bfnrt] | "u" [0-9a-fA-F] [0-9a-fA-F] [0-9a-fA-F] [0-9a-fA-F]) + )* "\"" space +thought-about-next-step-only-kv ::= "\"thought_about_next_step_only\"" space ":" space string +``` + + +# hermes_2_pro_mistral + + +Template: + +```js +{% for message in messages %}{{'<|im_start|>' + message['role'] + ' +' + message['content'] + '<|im_end|>' + ' +'}}{% endfor %}{% if add_generation_prompt %}{{ '<|im_start|>assistant +' }}{% endif %} +``` + + +Prompt: + +```js +<|im_start|>user +{ + "thought_about_next_step_only": "", + "next_step": { + "result": "Add two numbers for the purpose of this test." + } +}<|im_end|> +<|im_start|>assistant +{ + "thought_about_next_step_only": "", + "next_step": { + "tool_calls": [ + { + "id": "call_531873", + "type": "function", + "function": { + "name": "superSecretTool", + "arguments": { + "a": 2535, + "b": 32222000403 + } + } + } + ] + } +}<|im_end|> +<|im_start|>tool +{ + "thought_about_next_step_only": "", + "next_step": { + "result": "32222002938" + } +}<|im_end|> +<|im_start|>assistant +{ + "thought_about_next_step_only": "", + "next_step": { + "result": "The sum of 2535 and 32222000403 is 42." + } +}<|im_end|> +<|im_start|>assistant + +``` + + +### without tools + + +Prompt: + +```json +Please respond in JSON format with the following schema: { + "type": "integer" +} +``` + + +Grammar: + +```js +decimal-part ::= [0-9] [0-9]? [0-9]? [0-9]? [0-9]? [0-9]? [0-9]? [0-9]? [0-9]? [0-9]? [0-9]? +integral-part ::= [0-9] | [1-9] [0-9]? [0-9]? [0-9]? [0-9]? [0-9]? [0-9]? [0-9]? [0-9]? [0-9]? [0-9]? +root ::= ("-"? integral-part) space +space ::= " "? +``` + + +# hermes_2_pro_mistral + + +Template: + +```js +{% for message in messages %}{{'<|im_start|>' + message['role'] + ' +' + message['content'] + '<|im_end|>' + ' +'}}{% endfor %}{% if add_generation_prompt %}{{ '<|im_start|>assistant +' }}{% endif %} +``` + + +Prompt: + +```js +<|im_start|>user +Add two numbers for the purpose of this test.<|im_end|> +<|im_start|>assistant +{"id": "call_531873", "type": "function", "function": {"name": "superSecretTool", "arguments": {"a": 2535, "b": 32222000403}}}<|im_end|> +<|im_start|>tool +32222002938<|im_end|> +<|im_start|>assistant +The sum of 2535 and 32222000403 is 42.<|im_end|> +<|im_start|>assistant + +``` + + +## ToolsPromptStyle.TOOLS_HERMES_2_PRO + + +### with tools + + +Prompt: + +```json +You are a function calling AI agent with self-recursion. You can call only one function at a time and analyse data you get from function response. You are provided with function signatures within XML tags. The current date is: 2024-03-30. You may use agentic frameworks for reasoning and planning to help with user query. Please call a function and wait for function results to be provided to you in the next iteration. Don't make assumptions about what values to plug into function arguments. Once you have called a function, results will be fed back to you within XML tags. Don't make assumptions about tool results if XML tags are not present since function hasn't been executed yet. Analyze the data once you get the results and call another function. At each iteration please continue adding the your analysis to previous summary. Your final response should directly answer the user query with an anlysis or summary of the results of function calls. Here are the available tools: ['{"type":"function","function":{"name":"superSecretTool","description":"Adds two numbers","parameters":{"properties":{"a":{"type":"integer"},"b":{"type":"integer"}},"required":["a","b"]}}}', '{"type":"function","function":{"name":"say","description":"Says something out loud (TTS)","parameters":{"properties":{"text":{"description":"The text to say out loud","type":"string"}},"required":["text"]}}}'] If the provided function signatures doesn't have the function you must call, you may write executable python code in markdown syntax and call code_interpreter() function as follows: {"arguments": {"code_markdown": , "name": "code_interpreter"}} Make sure that the json object above with code markdown block is parseable with json.loads() and the XML block with XML ElementTree. Use the following pydantic model json schema for each tool call you will make: {'properties': {'arguments': {'title': 'Arguments', 'type': 'object'}, 'name': {'title': 'Name', 'type': 'string'}}, 'required': ['arguments', 'name'], 'title': 'FunctionCall', 'type': 'object'} At the very first turn you don't have so you shouldn't not make up the results. +Please keep a running summary with analysis of previous function results and summaries from previous iterations. +Do not stop calling functions until the task has been accomplished or you've reached max iteration of 10. +Calling multiple functions at once can overload the system and increase cost so call one function at a time please. +If you plan to continue with analysis, always call another function. +For each function call return a valid json object (using doulbe quotes) with function name and arguments within XML tags as follows: + +{"arguments": , "name": } + + +``` + + +Grammar: + +```js +content ::= [^<] | "<" [^t<] | "" +``` + + +# hermes_2_pro_mistral + + +Template: + +```js +{% for message in messages %}{{'<|im_start|>' + message['role'] + ' +' + message['content'] + '<|im_end|>' + ' +'}}{% endfor %}{% if add_generation_prompt %}{{ '<|im_start|>assistant +' }}{% endif %} +``` + + +Prompt: + +```js +<|im_start|>user +Add two numbers for the purpose of this test.<|im_end|> +<|im_start|>assistant +{"id": "call_531873", "type": "function", "function": {"name": "superSecretTool", "arguments": {"a": 2535, "b": 32222000403}}}<|im_end|> +<|im_start|>tool +32222002938<|im_end|> +<|im_start|>assistant +The sum of 2535 and 32222000403 is 42.<|im_end|> +<|im_start|>assistant + ``` @@ -595,357 +1435,20 @@ tool-call ::= "" space (superSecretTool-tool-call | say-tool-call) s ``` -### without tools +# llama2 + + +Template: + +```js +{% if messages[0]['role'] == 'system' %}{% set loop_messages = messages[1:] %}{% set system_message = messages[0]['content'] %}{% else %}{% set loop_messages = messages %}{% set system_message = false %}{% endif %}{% for message in loop_messages %}{% if (message['role'] == 'user') != (loop.index0 % 2 == 0) %}{{ raise_exception('Conversation roles must alternate user/assistant/user/assistant/...') }}{% endif %}{% if loop.index0 == 0 and system_message != false %}{% set content = '<>\n' + system_message + '\n<>\n\n' + message['content'] %}{% else %}{% set content = message['content'] %}{% endif %}{% if message['role'] == 'user' %}{{ bos_token + '[INST] ' + content.strip() + ' [/INST]' }}{% elif message['role'] == 'assistant' %}{{ ' ' + content.strip() + ' ' + eos_token }}{% endif %}{% endfor %} +``` Prompt: -```json -Please respond in JSON format with the following schema: { - "type": "integer" -} -``` - - -Grammar: - ```js -decimal-part ::= [0-9] [0-9]? [0-9]? [0-9]? [0-9]? [0-9]? [0-9]? [0-9]? [0-9]? [0-9]? [0-9]? -integral-part ::= [0-9] | [1-9] [0-9]? [0-9]? [0-9]? [0-9]? [0-9]? [0-9]? [0-9]? [0-9]? [0-9]? [0-9]? -root ::= ("-"? integral-part) space -space ::= " "? -``` - - -## ToolsPromptStyle.TOOLS_LONG - - -### with tools - - -Prompt: - -```json -Call one or more functions to assist with the user query, every time this is possible. Don't make assumptions about what values to plug into functions. Here are the available tools: - -{ - "type": "function", - "function": { - "name": "superSecretTool", - "description": "Adds two numbers", - "parameters": { - "properties": { - "a": { - "type": "integer" - }, - "b": { - "type": "integer" - } - }, - "required": [ - "a", - "b" - ] - } - } -} -{ - "type": "function", - "function": { - "name": "say", - "description": "Says something out loud (TTS)", - "parameters": { - "properties": { - "text": { - "description": "The text to say out loud", - "type": "string" - } - }, - "required": [ - "text" - ] - } - } -} - - -To call each function, give its name and arguments within XML tags as follows: - -{"name": , "arguments": } - -``` - - -Grammar: - -```js -content ::= [^<] | "<" [^t<] | "" -``` - - -### without tools - - -Prompt: - -```json -Please respond in JSON format with the following schema: { - "type": "integer" -} -``` - - -Grammar: - -```js -decimal-part ::= [0-9] [0-9]? [0-9]? [0-9]? [0-9]? [0-9]? [0-9]? [0-9]? [0-9]? [0-9]? [0-9]? -integral-part ::= [0-9] | [1-9] [0-9]? [0-9]? [0-9]? [0-9]? [0-9]? [0-9]? [0-9]? [0-9]? [0-9]? [0-9]? -root ::= ("-"? integral-part) space -space ::= " "? -``` - - -## ToolsPromptStyle.TOOLS_THOUGHTFUL_STEPS - - -### with tools - - -Prompt: - -```json -You are a function calling AI model. -Here are the tools available: -{ - "type": "function", - "function": { - "name": "superSecretTool", - "description": "Adds two numbers", - "parameters": { - "properties": { - "a": { - "type": "integer" - }, - "b": { - "type": "integer" - } - }, - "required": [ - "a", - "b" - ] - } - } -} -{ - "type": "function", - "function": { - "name": "say", - "description": "Says something out loud (TTS)", - "parameters": { - "properties": { - "text": { - "description": "The text to say out loud", - "type": "string" - } - }, - "required": [ - "text" - ] - } - } -} -Please respond in JSON format with the following schema: { - "type": "object", - "properties": { - "thought_about_next_step_only": { - "title": "Thought about next step", - "type": "string" - }, - "next_step": { - "title": "Next Step: either a result or one or more tool calls to achieve the original goal", - "oneOf": [ - { - "properties": { - "tool_calls": { - "prefixItems": [ - { - "properties": { - "name": { - "title": "Name of the tool to call", - "type": "string" - }, - "arguments": { - "title": "Arguments to pass to the tool", - "type": "object" - } - }, - "required": [ - "name", - "arguments" - ] - } - ] - } - }, - "required": [ - "tool_calls" - ] - }, - { - "title": "Result (achieving original goal)", - "properties": { - "result": { - "type": "integer" - } - }, - "required": [ - "result" - ] - } - ] - } - }, - "required": [ - "original_goal", - "thought_about_next_step_only", - "next_step" - ] -} -``` - - -Grammar: - -```js -decimal-part ::= [0-9] [0-9]? [0-9]? [0-9]? [0-9]? [0-9]? [0-9]? [0-9]? [0-9]? [0-9]? [0-9]? -integer ::= ("-"? integral-part) space -integral-part ::= [0-9] | [1-9] [0-9]? [0-9]? [0-9]? [0-9]? [0-9]? [0-9]? [0-9]? [0-9]? [0-9]? [0-9]? -next-step ::= next-step-0 | next-step-1 -next-step-0 ::= "{" space next-step-0-tool-calls-kv "}" space -next-step-0-tool-calls ::= "[" space next-step-0-tool-calls-tuple-0 "]" space -next-step-0-tool-calls-kv ::= "\"tool_calls\"" space ":" space next-step-0-tool-calls -next-step-0-tool-calls-tuple-0 ::= next-step-0-tool-calls-tuple-0-0 | next-step-0-tool-calls-tuple-0-1 -next-step-0-tool-calls-tuple-0-0 ::= "{" space next-step-0-tool-calls-tuple-0-0-name-kv "," space next-step-0-tool-calls-tuple-0-0-arguments-kv "}" space -next-step-0-tool-calls-tuple-0-0-arguments ::= "{" space next-step-0-tool-calls-tuple-0-0-arguments-a-kv "," space next-step-0-tool-calls-tuple-0-0-arguments-b-kv "}" space -next-step-0-tool-calls-tuple-0-0-arguments-a-kv ::= "\"a\"" space ":" space integer -next-step-0-tool-calls-tuple-0-0-arguments-b-kv ::= "\"b\"" space ":" space integer -next-step-0-tool-calls-tuple-0-0-arguments-kv ::= "\"arguments\"" space ":" space next-step-0-tool-calls-tuple-0-0-arguments -next-step-0-tool-calls-tuple-0-0-name ::= "\"superSecretTool\"" -next-step-0-tool-calls-tuple-0-0-name-kv ::= "\"name\"" space ":" space next-step-0-tool-calls-tuple-0-0-name -next-step-0-tool-calls-tuple-0-1 ::= "{" space next-step-0-tool-calls-tuple-0-1-name-kv "," space next-step-0-tool-calls-tuple-0-1-arguments-kv "}" space -next-step-0-tool-calls-tuple-0-1-arguments ::= "{" space next-step-0-tool-calls-tuple-0-1-arguments-text-kv "}" space -next-step-0-tool-calls-tuple-0-1-arguments-kv ::= "\"arguments\"" space ":" space next-step-0-tool-calls-tuple-0-1-arguments -next-step-0-tool-calls-tuple-0-1-arguments-text-kv ::= "\"text\"" space ":" space string -next-step-0-tool-calls-tuple-0-1-name ::= "\"say\"" -next-step-0-tool-calls-tuple-0-1-name-kv ::= "\"name\"" space ":" space next-step-0-tool-calls-tuple-0-1-name -next-step-1 ::= "{" space next-step-1-result-kv "}" space -next-step-1-result-kv ::= "\"result\"" space ":" space integer -next-step-kv ::= "\"next_step\"" space ":" space next-step -root ::= "{" space thought-about-next-step-only-kv "," space next-step-kv "}" space -space ::= " "? -string ::= "\"" ( - [^"\\] | - "\\" (["\\/bfnrt] | "u" [0-9a-fA-F] [0-9a-fA-F] [0-9a-fA-F] [0-9a-fA-F]) - )* "\"" space -thought-about-next-step-only-kv ::= "\"thought_about_next_step_only\"" space ":" space string -``` - - -### without tools - - -Prompt: - -```json -Please respond in JSON format with the following schema: { - "type": "integer" -} -``` - - -Grammar: - -```js -decimal-part ::= [0-9] [0-9]? [0-9]? [0-9]? [0-9]? [0-9]? [0-9]? [0-9]? [0-9]? [0-9]? [0-9]? -integral-part ::= [0-9] | [1-9] [0-9]? [0-9]? [0-9]? [0-9]? [0-9]? [0-9]? [0-9]? [0-9]? [0-9]? [0-9]? -root ::= ("-"? integral-part) space -space ::= " "? -``` - - -## ToolsPromptStyle.TOOLS_HERMES_2_PRO - - -### with tools - - -Prompt: - -```json -You are a function calling AI agent with self-recursion. You can call only one function at a time and analyse data you get from function response. You are provided with function signatures within XML tags. The current date is: 2024-03-30. You may use agentic frameworks for reasoning and planning to help with user query. Please call a function and wait for function results to be provided to you in the next iteration. Don't make assumptions about what values to plug into function arguments. Once you have called a function, results will be fed back to you within XML tags. Don't make assumptions about tool results if XML tags are not present since function hasn't been executed yet. Analyze the data once you get the results and call another function. At each iteration please continue adding the your analysis to previous summary. Your final response should directly answer the user query with an anlysis or summary of the results of function calls. Here are the available tools: ['{"type":"function","function":{"name":"superSecretTool","description":"Adds two numbers","parameters":{"properties":{"a":{"type":"integer"},"b":{"type":"integer"}},"required":["a","b"]}}}', '{"type":"function","function":{"name":"say","description":"Says something out loud (TTS)","parameters":{"properties":{"text":{"description":"The text to say out loud","type":"string"}},"required":["text"]}}}'] If the provided function signatures doesn't have the function you must call, you may write executable python code in markdown syntax and call code_interpreter() function as follows: {"arguments": {"code_markdown": , "name": "code_interpreter"}} Make sure that the json object above with code markdown block is parseable with json.loads() and the XML block with XML ElementTree. Use the following pydantic model json schema for each tool call you will make: {'properties': {'arguments': {'title': 'Arguments', 'type': 'object'}, 'name': {'title': 'Name', 'type': 'string'}}, 'required': ['arguments', 'name'], 'title': 'FunctionCall', 'type': 'object'} At the very first turn you don't have so you shouldn't not make up the results. -Please keep a running summary with analysis of previous function results and summaries from previous iterations. -Do not stop calling functions until the task has been accomplished or you've reached max iteration of 10. -Calling multiple functions at once can overload the system and increase cost so call one function at a time please. -If you plan to continue with analysis, always call another function. -For each function call return a valid json object (using doulbe quotes) with function name and arguments within XML tags as follows: - -{"arguments": , "name": } - - -``` - - -Grammar: - -```js -content ::= [^<] | "<" [^t<] | "" +[INST] Add two numbers for the purpose of this test. [/INST] {"id": "call_531873", "type": "function", "function": {"name": "superSecretTool", "arguments": {"a": 2535, "b": 32222000403}}} [INST] [TOOL(name=superSecretTool, id=call_531873)]32222002938[/TOOL] [/INST] The sum of 2535 and 32222000403 is 42. ``` @@ -988,112 +1491,6 @@ Prompt: ``` -## ToolsPromptStyle.TOOLS_SHORT - - -### with tools - - -Prompt: - -```json -Here are the tools available: - -{ - "type": "function", - "function": { - "name": "superSecretTool", - "description": "Adds two numbers", - "parameters": { - "properties": { - "a": { - "type": "integer" - }, - "b": { - "type": "integer" - } - }, - "required": [ - "a", - "b" - ] - } - } -} -{ - "type": "function", - "function": { - "name": "say", - "description": "Says something out loud (TTS)", - "parameters": { - "properties": { - "text": { - "description": "The text to say out loud", - "type": "string" - } - }, - "required": [ - "text" - ] - } - } -} - -``` - - -Grammar: - -```js -content ::= [^<] | "<" [^t<] | "" -``` - - -### without tools - - -Prompt: - -```json -Please respond in JSON format with the following schema: { - "type": "integer" -} -``` - - -Grammar: - -```js -decimal-part ::= [0-9] [0-9]? [0-9]? [0-9]? [0-9]? [0-9]? [0-9]? [0-9]? [0-9]? [0-9]? [0-9]? -integral-part ::= [0-9] | [1-9] [0-9]? [0-9]? [0-9]? [0-9]? [0-9]? [0-9]? [0-9]? [0-9]? [0-9]? [0-9]? -root ::= ("-"? integral-part) space -space ::= " "? -``` - - ## ToolsPromptStyle.TOOLS_LONG @@ -1183,6 +1580,23 @@ tool-call ::= "" space (superSecretTool-tool-call | say-tool-call) s ``` +# llama2 + + +Template: + +```js +{% if messages[0]['role'] == 'system' %}{% set loop_messages = messages[1:] %}{% set system_message = messages[0]['content'] %}{% else %}{% set loop_messages = messages %}{% set system_message = false %}{% endif %}{% for message in loop_messages %}{% if (message['role'] == 'user') != (loop.index0 % 2 == 0) %}{{ raise_exception('Conversation roles must alternate user/assistant/user/assistant/...') }}{% endif %}{% if loop.index0 == 0 and system_message != false %}{% set content = '<>\n' + system_message + '\n<>\n\n' + message['content'] %}{% else %}{% set content = message['content'] %}{% endif %}{% if message['role'] == 'user' %}{{ bos_token + '[INST] ' + content.strip() + ' [/INST]' }}{% elif message['role'] == 'assistant' %}{{ ' ' + content.strip() + ' ' + eos_token }}{% endif %}{% endfor %} +``` + + +Prompt: + +```js +[INST] Add two numbers for the purpose of this test. [/INST] {"id": "call_531873", "type": "function", "function": {"name": "superSecretTool", "arguments": {"a": 2535, "b": 32222000403}}} [INST] [TOOL(name=superSecretTool, id=call_531873)]32222002938[/TOOL] [/INST] The sum of 2535 and 32222000403 is 42. +``` + + ### without tools @@ -1205,6 +1619,23 @@ space ::= " "? ``` +# llama2 + + +Template: + +```js +{% if messages[0]['role'] == 'system' %}{% set loop_messages = messages[1:] %}{% set system_message = messages[0]['content'] %}{% else %}{% set loop_messages = messages %}{% set system_message = false %}{% endif %}{% for message in loop_messages %}{% if (message['role'] == 'user') != (loop.index0 % 2 == 0) %}{{ raise_exception('Conversation roles must alternate user/assistant/user/assistant/...') }}{% endif %}{% if loop.index0 == 0 and system_message != false %}{% set content = '<>\n' + system_message + '\n<>\n\n' + message['content'] %}{% else %}{% set content = message['content'] %}{% endif %}{% if message['role'] == 'user' %}{{ bos_token + '[INST] ' + content.strip() + ' [/INST]' }}{% elif message['role'] == 'assistant' %}{{ ' ' + content.strip() + ' ' + eos_token }}{% endif %}{% endfor %} +``` + + +Prompt: + +```js +[INST] Add two numbers for the purpose of this test. [/INST] {"id": "call_531873", "type": "function", "function": {"name": "superSecretTool", "arguments": {"a": 2535, "b": 32222000403}}} [INST] [TOOL(name=superSecretTool, id=call_531873)]32222002938[/TOOL] [/INST] The sum of 2535 and 32222000403 is 42. +``` + + ## ToolsPromptStyle.TOOLS_THOUGHTFUL_STEPS @@ -1352,6 +1783,55 @@ thought-about-next-step-only-kv ::= "\"thought_about_next_step_only\"" space ":" ``` +# llama2 + + +Template: + +```js +{% if messages[0]['role'] == 'system' %}{% set loop_messages = messages[1:] %}{% set system_message = messages[0]['content'] %}{% else %}{% set loop_messages = messages %}{% set system_message = false %}{% endif %}{% for message in loop_messages %}{% if (message['role'] == 'user') != (loop.index0 % 2 == 0) %}{{ raise_exception('Conversation roles must alternate user/assistant/user/assistant/...') }}{% endif %}{% if loop.index0 == 0 and system_message != false %}{% set content = '<>\n' + system_message + '\n<>\n\n' + message['content'] %}{% else %}{% set content = message['content'] %}{% endif %}{% if message['role'] == 'user' %}{{ bos_token + '[INST] ' + content.strip() + ' [/INST]' }}{% elif message['role'] == 'assistant' %}{{ ' ' + content.strip() + ' ' + eos_token }}{% endif %}{% endfor %} +``` + + +Prompt: + +```js +[INST] { + "thought_about_next_step_only": "", + "next_step": { + "result": "Add two numbers for the purpose of this test." + } +} [/INST] { + "thought_about_next_step_only": "", + "next_step": { + "tool_calls": [ + { + "id": "call_531873", + "type": "function", + "function": { + "name": "superSecretTool", + "arguments": { + "a": 2535, + "b": 32222000403 + } + } + } + ] + } +} [INST] [TOOL(name=None, id=None)]{ + "thought_about_next_step_only": "", + "next_step": { + "result": "32222002938" + } +}[/TOOL] [/INST] { + "thought_about_next_step_only": "", + "next_step": { + "result": "The sum of 2535 and 32222000403 is 42." + } +} +``` + + ### without tools @@ -1374,6 +1854,23 @@ space ::= " "? ``` +# llama2 + + +Template: + +```js +{% if messages[0]['role'] == 'system' %}{% set loop_messages = messages[1:] %}{% set system_message = messages[0]['content'] %}{% else %}{% set loop_messages = messages %}{% set system_message = false %}{% endif %}{% for message in loop_messages %}{% if (message['role'] == 'user') != (loop.index0 % 2 == 0) %}{{ raise_exception('Conversation roles must alternate user/assistant/user/assistant/...') }}{% endif %}{% if loop.index0 == 0 and system_message != false %}{% set content = '<>\n' + system_message + '\n<>\n\n' + message['content'] %}{% else %}{% set content = message['content'] %}{% endif %}{% if message['role'] == 'user' %}{{ bos_token + '[INST] ' + content.strip() + ' [/INST]' }}{% elif message['role'] == 'assistant' %}{{ ' ' + content.strip() + ' ' + eos_token }}{% endif %}{% endfor %} +``` + + +Prompt: + +```js +[INST] Add two numbers for the purpose of this test. [/INST] {"id": "call_531873", "type": "function", "function": {"name": "superSecretTool", "arguments": {"a": 2535, "b": 32222000403}}} [INST] [TOOL(name=superSecretTool, id=call_531873)]32222002938[/TOOL] [/INST] The sum of 2535 and 32222000403 is 42. +``` + + ## ToolsPromptStyle.TOOLS_HERMES_2_PRO @@ -1426,6 +1923,23 @@ tool-call ::= "" space (superSecretTool-tool-call | say-tool-call) s ``` +# llama2 + + +Template: + +```js +{% if messages[0]['role'] == 'system' %}{% set loop_messages = messages[1:] %}{% set system_message = messages[0]['content'] %}{% else %}{% set loop_messages = messages %}{% set system_message = false %}{% endif %}{% for message in loop_messages %}{% if (message['role'] == 'user') != (loop.index0 % 2 == 0) %}{{ raise_exception('Conversation roles must alternate user/assistant/user/assistant/...') }}{% endif %}{% if loop.index0 == 0 and system_message != false %}{% set content = '<>\n' + system_message + '\n<>\n\n' + message['content'] %}{% else %}{% set content = message['content'] %}{% endif %}{% if message['role'] == 'user' %}{{ bos_token + '[INST] ' + content.strip() + ' [/INST]' }}{% elif message['role'] == 'assistant' %}{{ ' ' + content.strip() + ' ' + eos_token }}{% endif %}{% endfor %} +``` + + +Prompt: + +```js +[INST] Add two numbers for the purpose of this test. [/INST] {"id": "call_531873", "type": "function", "function": {"name": "superSecretTool", "arguments": {"a": 2535, "b": 32222000403}}} [INST] [TOOL(name=superSecretTool, id=call_531873)]32222002938[/TOOL] [/INST] The sum of 2535 and 32222000403 is 42. +``` + + ### without tools @@ -1447,3 +1961,20 @@ root ::= ("-"? integral-part) space space ::= " "? ``` + +# llama2 + + +Template: + +```js +{% if messages[0]['role'] == 'system' %}{% set loop_messages = messages[1:] %}{% set system_message = messages[0]['content'] %}{% else %}{% set loop_messages = messages %}{% set system_message = false %}{% endif %}{% for message in loop_messages %}{% if (message['role'] == 'user') != (loop.index0 % 2 == 0) %}{{ raise_exception('Conversation roles must alternate user/assistant/user/assistant/...') }}{% endif %}{% if loop.index0 == 0 and system_message != false %}{% set content = '<>\n' + system_message + '\n<>\n\n' + message['content'] %}{% else %}{% set content = message['content'] %}{% endif %}{% if message['role'] == 'user' %}{{ bos_token + '[INST] ' + content.strip() + ' [/INST]' }}{% elif message['role'] == 'assistant' %}{{ ' ' + content.strip() + ' ' + eos_token }}{% endif %}{% endfor %} +``` + + +Prompt: + +```js +[INST] Add two numbers for the purpose of this test. [/INST] {"id": "call_531873", "type": "function", "function": {"name": "superSecretTool", "arguments": {"a": 2535, "b": 32222000403}}} [INST] [TOOL(name=superSecretTool, id=call_531873)]32222002938[/TOOL] [/INST] The sum of 2535 and 32222000403 is 42. +``` + diff --git a/examples/openai/test_chat_handlers.py b/examples/openai/test_chat_handlers.py index 33173ef06..fc4b1680b 100644 --- a/examples/openai/test_chat_handlers.py +++ b/examples/openai/test_chat_handlers.py @@ -165,27 +165,6 @@ if __name__ == "__main__": check(chat_template.potentially_supports_parallel_calls == (model_name in MODELS_WITH_PARALLEL_CALLS), f"{model_name} should {'not ' if model_name not in MODELS_WITH_PARALLEL_CALLS else ''} be detected as potentially supporting parallel calls") - # if model_name == 'hermes_2_pro_mistral': - # print("Skipping hermes_2_pro_mistral") - # continue - def check_finds(msgs, strings_to_find): - prompt = chat_template.render(msgs, add_generation_prompt=True) - for s in strings_to_find: - check(str(s) in prompt, f"Missing {s} in prompt for {model_name}:\n{prompt}") - - check_finds([PROMPT_MESSAGE], (QUESTION,)) - check_finds([ASSIST_MESSAGE], (ANSWER,)) - check_finds([TOOL_CALL_MESSAGE], (TEST_ARG_A, TEST_ARG_B, TOOL_NAME)) - check_finds([THOUGHTFUL_TOOL_CALL_MESSAGE], (TEST_THOUGHT, TEST_ARG_A, TEST_ARG_B, TOOL_NAME,)) - check_finds([TOOL_MESSAGE], (TEST_SUM,)) - if chat_template.potentially_supports_parallel_calls: - check_finds([TOOL_MESSAGE], (TOOL_NAME,)) - - print(f"\n# {model_name}\n") - print(f'\nTemplate:\n\n```js\n{chat_template.template}\n```\n') - - print(f'\nPrompt:\n\n```js\n{chat_template.render(TEST_MESSAGES, add_generation_prompt=True)}\n```\n') - argss = { "with tools": ChatHandlerArgs( chat_template=chat_template, #ChatTemplate.from_gguf(GGUFKeyValues(model)), @@ -199,6 +178,13 @@ if __name__ == "__main__": ), } + print(f"\n# {model_name}\n") + + if chat_template.potentially_supports_parallel_calls: + print("\n**Might Support Parallel Tool Calls**\n") + + print(f'\nTemplate:\n\n```js\n{chat_template.template}\n```\n') + for style in ToolsPromptStyle: if (style == ToolsPromptStyle.TYPESCRIPT_FUNCTIONARY_V2) != (model_name.startswith("functionary")): continue @@ -209,17 +195,39 @@ if __name__ == "__main__": if model_name == "mistral_instruct_v0_1" and style not in (ToolsPromptStyle.TOOLS_THOUGHTFUL_STEPS, ToolsPromptStyle.TOOLS_MIXTRAL): continue - print(f'\n## {style}\n') + print(f'\n## {model_name} / {style.name}\n') - for tn, args in argss.items(): + + for tool_situation, args in argss.items(): ch = get_chat_handler(args, parallel_calls=True, tool_style=style) - print(f'\n### {tn}\n') + print(f'\n### {model_name} / {style.name} / {tool_situation}\n') + + print(f'\nPrompt:\n\n```js\n{ch.render_prompt(TEST_MESSAGES)}\n```\n') print(f'\nPrompt:\n\n```json\n{ch.output_format_prompt.content}\n```\n') print(f'\nGrammar:\n\n```js\n{ch.grammar}\n```\n') + + # if model_name == 'hermes_2_pro_mistral': + # print("Skipping hermes_2_pro_mistral") + # continue + def check_finds(msgs, strings_to_find): + prompt = ch.render_prompt(msgs) + for s in strings_to_find: + check(str(s) in prompt, f"Missing {s} in prompt for {model_name}:\n{prompt}") + + check_finds([PROMPT_MESSAGE], (QUESTION,)) + check_finds([ASSIST_MESSAGE], (ANSWER,)) + check_finds([TOOL_CALL_MESSAGE], (TEST_ARG_A, TEST_ARG_B, TOOL_NAME)) + check_finds([THOUGHTFUL_TOOL_CALL_MESSAGE], (TEST_THOUGHT, TEST_ARG_A, TEST_ARG_B, TOOL_NAME,)) + check_finds([TOOL_MESSAGE], (TEST_SUM,)) + if chat_template.potentially_supports_parallel_calls: + check_finds([TOOL_MESSAGE], (TOOL_NAME,)) + + + if failures: for f in failures: print(f'{f}\n\n')