vbatts/llama.cpp
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Allow simpler function calling sytax, like used with Phi-3 function calling model

Browse source
This commit is contained in:
Don Mahurin 2024-09-28 14:10:55 -07:00
parent 8550b76f4e
commit b790a7ff29
3 changed files with 33 additions and 11 deletions
Download patch file Download diff file Expand all files Collapse all files

5
examples/function-calling/README.md
View file

@ -44,3 +44,8 @@ What is 37234 times 39?
To calculate 37234 times 39, I'll perform the multiplication. Let's do that.
The result of multiplying 37234 by 39 is 1,452,126. If you have any more calculations or questions, feel free to ask!
```
## Function calling example, using Phi-3 function calling
```
./examples/function-calling/llama-cli-function-runner.py -m `huggingface-cli download nold/Phi-3-mini-4k-instruct-function-calling-GGUF Phi-3-mini-4k-instruct-function-calling_Q4_K_M.gguf` --special --display-prompt -i
```

35
examples/function-calling/function_tool.py
View file

@ -64,6 +64,9 @@ def generate_functionary_schema_from_functions(functions, namespace="functions")
schema += "}} // namespace {}".format(namespace)
return schema
def generate_simple_schema_from_functions(functions) -> str:
return '\n'.join([json.dumps(function).replace('{', '{ ').replace('}', ' }') for function in functions])
functionary_prompt_start = """<|start_header_id|>system<|end_header_id|>
You are capable of executing available function(s) if required.
@ -81,13 +84,27 @@ functionary_prompt_end = """<|eot_id|><|start_header_id|>system<|end_header_id|>
When you send a message containing Python code to python, it will be executed in a stateful Jupyter notebook environment. python will respond with the output of the execution or time out after 60.0 seconds. The drive at '/mnt/data' can be used to save and persist user files.<|eot_id|><|start_header_id|>user<|end_header_id|>
"""
simple_prompt_start = """<s><|user|> You are a helpful assistant with access to the following functions. Use them if required - """
simple_prompt_end = """<|end|>"""
def get_chat_tool_format(args, tools):
return {
'prompt': functionary_prompt_start + generate_functionary_schema_from_functions(tools) + functionary_prompt_end,
'function_marker': '>>>',
'function_re': r'>>>([^\n]*)\n(.*)<\|eot_id\|>',
'user_start': '<|start_header_id|>user<|end_header_id|>\n',
'user_end': '<|eot_id|><|start_header_id|>assistant<|end_header_id|>' + '\n',
'tool_start': '',
'tool_end': '<|eot_id|><|start_header_id|>assistant<|end_header_id|>'
}
if 'functionary' in args.model.lower():
return {
'prompt': functionary_prompt_start + generate_functionary_schema_from_functions(tools) + functionary_prompt_end,
'function_marker': '>>>',
'function_re': r'>>>([^\n]*)\n(.*)<\|eot_id\|>',
'user_start': '<|start_header_id|>user<|end_header_id|>\n',
'user_end': '<|eot_id|><|start_header_id|>assistant<|end_header_id|>' + '\n',
'tool_start': '',
'tool_end': '<|eot_id|><|start_header_id|>assistant<|end_header_id|>'
}
else:
return {
'prompt': simple_prompt_start + generate_simple_schema_from_functions(tools) + simple_prompt_end,
'function_marker': '<functioncall>',
'function_re': r'<functioncall> \n?(.*)<\|end\|>',
'user_start': '<|user|> ',
'user_end': '<|end|>' + '\n',
'tool_start': '<|user|>',
'tool_end': '<|end|> <|assistant|>'
}

4
examples/function-calling/llama-cli-function-runner.py
View file

@ -23,6 +23,7 @@ def main():
parser.add_argument('--display-prompt', action=argparse.BooleanOptionalAction, default=False)
parser.add_argument('--special', action=argparse.BooleanOptionalAction, default=False)
parser.add_argument('--reverse-prompt', type=str)
parser.add_argument('-m', '--model', type=str, default='model.gguf')
parser.add_argument('--ctx-size', type=int, default=1024)
args, other_args = parser.parse_known_args()
@ -31,8 +32,7 @@ def main():
if args.display_prompt: print(tool_format['prompt'])
command = [ './llama-cli', '-i', '-p', tool_format['prompt'], '--reverse-prompt', args.reverse_prompt, '--escape', '--special', '--no-display-prompt', '--log-disable', '--simple-io', '--ctx-size', str(args.ctx_size), *other_args]
print("'" + "' '".join(command) + "'")
command = [ './llama-cli', '-i', '-p', tool_format['prompt'], '--model', args.model, '--reverse-prompt', args.reverse_prompt, '--escape', '--special', '--no-display-prompt', '--log-disable', '--simple-io', '--ctx-size', str(args.ctx_size), *other_args]
process = subprocess.Popen(
command,