diff --git a/examples/agent/README.md b/examples/agent/README.md new file mode 100644 index 000000000..fd5d37a71 --- /dev/null +++ b/examples/agent/README.md @@ -0,0 +1,33 @@ +# Agents / Tool Calling w/ llama.cpp + +- Install prerequisite: [uv](https://docs.astral.sh/uv/) (used to simplify python deps) + +- Run `llama-server` w/ jinja templates: + + ```bash + make -j LLAMA_CURL=1 llama-server + ./llama-server \ + -mu https://huggingface.co/lmstudio-community/Meta-Llama-3.1-70B-Instruct-GGUF/resolve/main/Meta-Llama-3.1-70B-Instruct-Q4_K_M.gguf \ + --jinja \ + -c 8192 -fa + ``` + +- Run some tools inside a docker container (check http://localhost:8088/docs once running): + + ```bash + docker run -p 8088:8088 -w /src \ + -v $PWD/examples/agent:/src \ + --rm -it ghcr.io/astral-sh/uv:python3.12-alpine \ + uv run fastify.py --port 8088 tools.py + ``` + + > [!WARNING] + > The command above gives tools (and your agent) access to the web (and read-only access to `examples/agent/**`. If you're concerned about unleashing a rogue agent on the web, please explore setting up proxies for your docker (and contribute back!) + +- Run the agent with a given goal: + + ```bash + uv run examples/agent/run.py \ + --tool-endpoint http://localhost:8088 \ + --goal "What is the sum of 2535 squared and 32222000403?" + ``` diff --git a/examples/tool-call/fastify.py b/examples/agent/fastify.py similarity index 100% rename from examples/tool-call/fastify.py rename to examples/agent/fastify.py diff --git a/examples/tool-call/agent.py b/examples/agent/run.py similarity index 95% rename from examples/tool-call/agent.py rename to examples/agent/run.py index 8e545a82d..edccc5aa5 100644 --- a/examples/tool-call/agent.py +++ b/examples/agent/run.py @@ -22,6 +22,9 @@ import urllib.parse class OpenAPIMethod: def __init__(self, url, name, descriptor, catalog): + ''' + Wraps a remote OpenAPI method as a Python function. + ''' self.url = url self.__name__ = name @@ -96,11 +99,8 @@ def main( goal: Annotated[str, typer.Option()], api_key: Optional[str] = None, tool_endpoint: Optional[list[str]] = None, - format: Annotated[Optional[str], typer.Option(help="The output format: either a Python type (e.g. 'float' or a Pydantic model defined in one of the tool files), or a JSON schema, e.g. '{\"format\": \"date\"}'")] = None, max_iterations: Optional[int] = 10, - parallel_calls: Optional[bool] = False, verbose: bool = False, - # endpoint: Optional[str] = None, endpoint: str = "http://localhost:8080/v1/", ): @@ -110,6 +110,7 @@ def main( tool_map = {} tools = [] + # Discover tools using OpenAPI catalogs at the provided endpoints. for url in (tool_endpoint or []): assert url.startswith('http://') or url.startswith('https://'), f'Tools must be URLs, not local files: {url}' diff --git a/examples/tool-call/tools.py b/examples/agent/tools.py similarity index 53% rename from examples/tool-call/tools.py rename to examples/agent/tools.py index 0d630234a..6c4479ef9 100644 --- a/examples/tool-call/tools.py +++ b/examples/agent/tools.py @@ -1,3 +1,9 @@ +# /// script +# requires-python = ">=3.10" +# dependencies = [ +# "ipython", +# ] +# /// import datetime import json from pydantic import BaseModel @@ -82,32 +88,69 @@ def _is_serializable(obj) -> bool: except Exception as e: return False -def python(source: str) -> Union[Dict, str]: +def python(code: str) -> str: """ - Evaluate a Python program and return the globals it declared. - Can be used to compute mathematical expressions (e.g. after importing math module). - Args: - source: contain valid, executable and pure Python code. Should also import any required Python packages. - For example: "import math\nresult = math.cos(2) * 10" - Returns: - dict | str: A dictionary containing variables declared, or an error message if an exception occurred. + Executes Python code in a siloed environment using IPython and returns the output. + + Parameters: + code (str): The Python code to execute. + + Returns: + str: The output of the executed code. """ + from IPython import InteractiveShell + from io import StringIO + import sys + + # Create an isolated IPython shell instance + shell = InteractiveShell() + + # Redirect stdout to capture output + old_stdout = sys.stdout + sys.stdout = mystdout = StringIO() + try: - namespace = {} - sys.stderr.write(f"Executing Python program:\n{source}\n") - exec(source, namespace) - results = { - k: v - for k, v in namespace.items() - if not k.startswith('_') \ - and not isinstance(v, type) \ - and not isinstance(v, types.ModuleType) \ - and not callable(v) \ - and _is_serializable(v) - } - sys.stderr.write(f"Results: {json.dumps(results, indent=2)}\n") - return results + # Execute the code + shell.run_cell(code) except Exception as e: - msg = f"Error: {sys.exc_info()[1]}" - sys.stderr.write(f"{msg}\n") - return msg + # Restore stdout before returning + sys.stdout = old_stdout + return f"An error occurred: {e}" + finally: + # Always restore stdout + sys.stdout = old_stdout + + # Retrieve the output + output = mystdout.getvalue() + return output + + +# def python(source: str) -> Union[Dict, str]: +# """ +# Evaluate a Python program and return the globals it declared. +# Can be used to compute mathematical expressions (e.g. after importing math module). +# Args: +# source: contain valid, executable and pure Python code. Should also import any required Python packages. +# For example: "import math\nresult = math.cos(2) * 10" +# Returns: +# dict | str: A dictionary containing variables declared, or an error message if an exception occurred. +# """ +# try: +# namespace = {} +# sys.stderr.write(f"Executing Python program:\n{source}\n") +# exec(source, namespace) +# results = { +# k: v +# for k, v in namespace.items() +# if not k.startswith('_') \ +# and not isinstance(v, type) \ +# and not isinstance(v, types.ModuleType) \ +# and not callable(v) \ +# and _is_serializable(v) +# } +# sys.stderr.write(f"Results: {json.dumps(results, indent=2)}\n") +# return results +# except Exception as e: +# msg = f"Error: {sys.exc_info()[1]}" +# sys.stderr.write(f"{msg}\n") +# return msg diff --git a/examples/tool-call/README.md b/examples/tool-call/README.md deleted file mode 100644 index e6c689ebe..000000000 --- a/examples/tool-call/README.md +++ /dev/null @@ -1,33 +0,0 @@ -# Agents / Tool Calling w/ llama.cpp - -- Install prerequisite: [uv](https://docs.astral.sh/uv/) (used to simplify python deps) - -- Run `llama-server` w/ jinja templates: - - ```bash - # make -j LLAMA_CURL=1 llama-server - ./llama-server \ - -mu https://huggingface.co/lmstudio-community/Meta-Llama-3.1-70B-Instruct-GGUF/resolve/main/Meta-Llama-3.1-70B-Instruct-Q4_K_M.gguf \ - --jinja \ - -c 8192 -fa - ``` - -- Run some tools inside a docker container - - ```bash - docker run --rm -it \ - -p "8088:8088" \ - -v $PWD/examples/tool-call:/src \ - ghcr.io/astral-sh/uv:python3.12-alpine \ - uv run /src/fastify.py --port 8088 /src/tools.py - ``` - -- Verify which tools have been exposed: http://localhost:8088/docs - -- Run the agent with a given goal: - - ```bash - uv run examples/tool-call/agent.py \ - --tool-endpoint http://localhost:8088 \ - --goal "What is the sum of 2535 squared and 32222000403 then multiplied by one and a half. What's a third of the result?" - ``` diff --git a/requirements.txt b/requirements.txt index 9e190ae27..8543d5e6b 100644 --- a/requirements.txt +++ b/requirements.txt @@ -10,3 +10,5 @@ -r ./requirements/requirements-convert_hf_to_gguf_update.txt -r ./requirements/requirements-convert_llama_ggml_to_gguf.txt -r ./requirements/requirements-convert_lora_to_gguf.txt + +-r ./requirements/requirements-agent.txt diff --git a/requirements/requirements-agent.txt b/requirements/requirements-agent.txt new file mode 100644 index 000000000..639f0111f --- /dev/null +++ b/requirements/requirements-agent.txt @@ -0,0 +1,6 @@ +fastapi +openai +pydantic +requests +typer +uvicorn