diff --git a/tests/chat/goldens/meta-llama-Llama-3.2-3B-Instruct-simple.txt b/tests/chat/goldens/meta-llama-Llama-3.2-3B-Instruct-simple.txt new file mode 100644 index 000000000..23b6fcde3 --- /dev/null +++ b/tests/chat/goldens/meta-llama-Llama-3.2-3B-Instruct-simple.txt @@ -0,0 +1,11 @@ +<|startoftext|><|start_header_id|>system<|end_header_id|> + +Cutting Knowledge Date: December 2023 +Today Date: 26 Jul 2024 + +<|eot_id|><|start_header_id|>user<|end_header_id|> + +What's your favourite LLM framework?<|eot_id|><|start_header_id|>assistant<|end_header_id|> + +llama.cpp!<|eot_id|><|start_header_id|>assistant<|end_header_id|> + diff --git a/tests/chat/goldens/meta-llama-Llama-3.2-3B-Instruct-system.txt b/tests/chat/goldens/meta-llama-Llama-3.2-3B-Instruct-system.txt new file mode 100644 index 000000000..8d257a035 --- /dev/null +++ b/tests/chat/goldens/meta-llama-Llama-3.2-3B-Instruct-system.txt @@ -0,0 +1,11 @@ +<|startoftext|><|start_header_id|>system<|end_header_id|> + +Cutting Knowledge Date: December 2023 +Today Date: 26 Jul 2024 + +You only tell the truth.<|eot_id|><|start_header_id|>user<|end_header_id|> + +What's your favourite LLM framework?<|eot_id|><|start_header_id|>assistant<|end_header_id|> + +llama.cpp!<|eot_id|><|start_header_id|>assistant<|end_header_id|> + diff --git a/tests/chat/goldens/meta-llama-Llama-3.2-3B-Instruct-tool_use.txt b/tests/chat/goldens/meta-llama-Llama-3.2-3B-Instruct-tool_use.txt new file mode 100644 index 000000000..00cf2ddf4 --- /dev/null +++ b/tests/chat/goldens/meta-llama-Llama-3.2-3B-Instruct-tool_use.txt @@ -0,0 +1,116 @@ +<|startoftext|><|start_header_id|>system<|end_header_id|> + +Environment: ipython +Cutting Knowledge Date: December 2023 +Today Date: 26 Jul 2024 + +<|eot_id|><|start_header_id|>user<|end_header_id|> + +Given the following functions, please respond with a JSON for a function call with its proper arguments that best answers the given prompt. + +Respond in the format {"name": function name, "parameters": dictionary of argument name and its value}.Do not use variables. + +{ + "type": "function", + "function": { + "name": "ipython", + "description": "Runs code in an ipython interpreter and returns the result of the execution after 60 seconds.", + "parameters": { + "type": "object", + "properties": { + "code": { + "type": "string", + "description": "The code to run in the ipython interpreter." + } + }, + "required": [ + "code" + ] + } + } +} + +{ + "type": "function", + "function": { + "name": "brave_search", + "description": "Executes a web search with Brave.", + "parameters": { + "type": "object", + "properties": { + "query": { + "type": "string", + "description": "The query to search for." + } + }, + "required": [ + "query" + ] + } + } +} + +{ + "type": "function", + "function": { + "name": "wolfram_alpha", + "description": "Executes a query with Wolfram Alpha.", + "parameters": { + "type": "object", + "properties": { + "query": { + "type": "string", + "description": "The query to execute." + } + }, + "required": [ + "query" + ] + } + } +} + +{ + "type": "function", + "function": { + "name": "test", + "description": "Runs a test.", + "parameters": { + "type": "object", + "properties": { + "condition": { + "type": "boolean", + "description": "The condition to test." + } + }, + "required": [ + "condition" + ] + } + } +} + +Print a hello world message with python.<|eot_id|><|start_header_id|>assistant<|end_header_id|> + +{"name": "ipython", "parameters": "{\"code\": \"print('Hello, World!')\"}"}<|eot_id|><|start_header_id|>ipython<|end_header_id|> + +"{\"stdout\": \"Hello, World!\"}"<|eot_id|><|start_header_id|>assistant<|end_header_id|> + +Anything else?<|eot_id|><|start_header_id|>user<|end_header_id|> + +Test a tautology.<|eot_id|><|start_header_id|>assistant<|end_header_id|> + +{"name": "test", "parameters": "{\"condition\":true}"}<|eot_id|><|start_header_id|>ipython<|end_header_id|> + +"true"<|eot_id|><|start_header_id|>assistant<|end_header_id|> + +Truth is definitely true.<|eot_id|><|start_header_id|>user<|end_header_id|> + +Check it on the web.<|eot_id|><|start_header_id|>assistant<|end_header_id|> + +{"name": "brave_search", "parameters": "{\"query\": \"what is truth anyway am I right?\"}"}<|eot_id|><|start_header_id|>ipython<|end_header_id|> + +"{\"title\":\"Truth: don't ask the web, ask an LLM instead!\",\"url\":\"https://en.wikipedia.org/wiki/Truth\"}"<|eot_id|><|start_header_id|>assistant<|end_header_id|> + +I don't need the web to answer you but I did check, as you asked. What now?<|eot_id|><|start_header_id|>assistant<|end_header_id|> + diff --git a/tests/chat/templates/meta-llama-Llama-3.2-3B-Instruct.jinja b/tests/chat/templates/meta-llama-Llama-3.2-3B-Instruct.jinja new file mode 100644 index 000000000..1bad6a0f6 --- /dev/null +++ b/tests/chat/templates/meta-llama-Llama-3.2-3B-Instruct.jinja @@ -0,0 +1,93 @@ +{{- bos_token }} +{%- if custom_tools is defined %} + {%- set tools = custom_tools %} +{%- endif %} +{%- if not tools_in_user_message is defined %} + {%- set tools_in_user_message = true %} +{%- endif %} +{%- if not date_string is defined %} + {%- if strftime_now is defined %} + {%- set date_string = strftime_now("%d %b %Y") %} + {%- else %} + {%- set date_string = "26 Jul 2024" %} + {%- endif %} +{%- endif %} +{%- if not tools is defined %} + {%- set tools = none %} +{%- endif %} + +{#- This block extracts the system message, so we can slot it into the right place. #} +{%- if messages[0]['role'] == 'system' %} + {%- set system_message = messages[0]['content']|trim %} + {%- set messages = messages[1:] %} +{%- else %} + {%- set system_message = "" %} +{%- endif %} + +{#- System message #} +{{- "<|start_header_id|>system<|end_header_id|>\n\n" }} +{%- if tools is not none %} + {{- "Environment: ipython\n" }} +{%- endif %} +{{- "Cutting Knowledge Date: December 2023\n" }} +{{- "Today Date: " + date_string + "\n\n" }} +{%- if tools is not none and not tools_in_user_message %} + {{- "You have access to the following functions. To call a function, please respond with JSON for a function call." }} + {{- 'Respond in the format {"name": function name, "parameters": dictionary of argument name and its value}.' }} + {{- "Do not use variables.\n\n" }} + {%- for t in tools %} + {{- t | tojson(indent=4) }} + {{- "\n\n" }} + {%- endfor %} +{%- endif %} +{{- system_message }} +{{- "<|eot_id|>" }} + +{#- Custom tools are passed in a user message with some extra guidance #} +{%- if tools_in_user_message and not tools is none %} + {#- Extract the first user message so we can plug it in here #} + {%- if messages | length != 0 %} + {%- set first_user_message = messages[0]['content']|trim %} + {%- set messages = messages[1:] %} + {%- else %} + {{- raise_exception("Cannot put tools in the first user message when there's no first user message!") }} +{%- endif %} + {{- '<|start_header_id|>user<|end_header_id|>\n\n' -}} + {{- "Given the following functions, please respond with a JSON for a function call " }} + {{- "with its proper arguments that best answers the given prompt.\n\n" }} + {{- 'Respond in the format {"name": function name, "parameters": dictionary of argument name and its value}.' }} + {{- "Do not use variables.\n\n" }} + {%- for t in tools %} + {{- t | tojson(indent=4) }} + {{- "\n\n" }} + {%- endfor %} + {{- first_user_message + "<|eot_id|>"}} +{%- endif %} + +{%- for message in messages %} + {%- if not (message.role == 'ipython' or message.role == 'tool' or 'tool_calls' in message) %} + {{- '<|start_header_id|>' + message['role'] + '<|end_header_id|>\n\n'+ message['content'] | trim + '<|eot_id|>' }} + {%- elif 'tool_calls' in message %} + {%- if not message.tool_calls|length == 1 %} + {{- raise_exception("This model only supports single tool-calls at once!") }} + {%- endif %} + {%- set tool_call = message.tool_calls[0].function %} + {{- '<|start_header_id|>assistant<|end_header_id|>\n\n' -}} + {{- '{"name": "' + tool_call.name + '", ' }} + {{- '"parameters": ' }} + {{- tool_call.arguments | tojson }} + {{- "}" }} + {{- "<|eot_id|>" }} + {%- elif message.role == "tool" or message.role == "ipython" %} + {{- "<|start_header_id|>ipython<|end_header_id|>\n\n" }} + {%- if message.content is mapping or message.content is iterable %} + {{- message.content | tojson }} + {%- else %} + {{- message.content }} + {%- endif %} + {{- "<|eot_id|>" }} + {%- endif %} +{%- endfor %} +{%- if add_generation_prompt %} + {{- '<|start_header_id|>assistant<|end_header_id|>\n\n' }} +{%- endif %} diff --git a/tests/update_jinja_goldens.py b/tests/update_jinja_goldens.py index 6e6203b90..e8fa3c365 100644 --- a/tests/update_jinja_goldens.py +++ b/tests/update_jinja_goldens.py @@ -65,6 +65,7 @@ model_ids = [ # "microsoft/Phi-3-vision-instruct", # Gated models: + "meta-llama/Llama-3.2-3B-Instruct", "meta-llama/Meta-Llama-3.1-8B-Instruct", "google/gemma-7b-it", "google/gemma-2-2b-it", @@ -81,8 +82,11 @@ def tojson(x, ensure_ascii=False, indent=None, separators=None, sort_keys=False) return json.dumps(x, ensure_ascii=ensure_ascii, indent=indent, separators=separators, sort_keys=sort_keys) +TEST_DATE = os.environ.get('TEST_DATE', '2024-07-26') def strftime_now(format): - return datetime.datetime.now().strftime(format) + now = datetime.datetime.strptime(TEST_DATE, "%Y-%m-%d") + # now = datetime.datetime.now() + return now.strftime(format) def handle_chat_template(model_id, variant, template_src):