From ffc74360e211777b32e418a8c0e54f088d82f77d Mon Sep 17 00:00:00 2001 From: ochafik Date: Tue, 26 Mar 2024 01:26:45 +0000 Subject: [PATCH] agents: scripts to run scripts as sandboxed fastapi servers --- .gitmodules | 3 + examples/agents/README.md | 15 +++++ examples/agents/fastify-requirements.txt | 5 ++ examples/agents/fastify.py | 63 ++++++++++++++++++ examples/agents/hermes_function_calling | 1 + examples/agents/requirements.txt | 3 + examples/agents/run_sandboxed_tools.sh | 82 ++++++++++++++++++++++++ 7 files changed, 172 insertions(+) create mode 100644 examples/agents/README.md create mode 100644 examples/agents/fastify-requirements.txt create mode 100644 examples/agents/fastify.py create mode 160000 examples/agents/hermes_function_calling create mode 100644 examples/agents/requirements.txt create mode 100755 examples/agents/run_sandboxed_tools.sh diff --git a/.gitmodules b/.gitmodules index b7e8b8ff2..9d262566c 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,3 +1,6 @@ [submodule "kompute"] path = kompute url = https://github.com/nomic-ai/kompute.git +[submodule "examples/agents/hermes_function_calling"] + path = examples/agents/hermes_function_calling + url = https://github.com/NousResearch/Hermes-Function-Calling diff --git a/examples/agents/README.md b/examples/agents/README.md new file mode 100644 index 000000000..eb743d0c1 --- /dev/null +++ b/examples/agents/README.md @@ -0,0 +1,15 @@ + +Edit `examples/agents/hermes_function_calling/utils.py`: + +```py +log_folder = os.environ.get('LOG_FOLDER', os.path.join(script_dir, "inference_logs")) +``` + +Then run: + +```bash +REQUIREMENTS_FILE=<( cat examples/agents/hermes_function_calling/requirements.txt | grep -vE "bitsandbytes|flash-attn" ) \ + examples/agents/run_sandboxed_tools.sh \ + examples/agents/hermes_function_calling/functions.py \ + -e LOG_FOLDER=/data/inference_logs +``` \ No newline at end of file diff --git a/examples/agents/fastify-requirements.txt b/examples/agents/fastify-requirements.txt new file mode 100644 index 000000000..abd7fe8d1 --- /dev/null +++ b/examples/agents/fastify-requirements.txt @@ -0,0 +1,5 @@ +fastapi[all] +pydantic +sse-starlette +uvicorn[all] +typer[all] \ No newline at end of file diff --git a/examples/agents/fastify.py b/examples/agents/fastify.py new file mode 100644 index 000000000..48df2bfda --- /dev/null +++ b/examples/agents/fastify.py @@ -0,0 +1,63 @@ +''' + Binds the functions of a python script as a FastAPI server. + + This is useful in combination w/ the examples/agent/run_sandboxed_tools.sh +''' +import os, sys, typing, importlib.util +from anyio import Path +import fastapi, uvicorn +import typer + +# from langchain_core.tools import BaseTool + +def load_source_as_module(source): + i = 0 + while (module_name := f'mod_{i}') in sys.modules: + i += 1 + + spec = importlib.util.spec_from_file_location(module_name, source) + module = importlib.util.module_from_spec(spec) + sys.modules[module_name] = module + spec.loader.exec_module(module) + return module + +def bind_functions(app, module): + for k in dir(module): + if k.startswith('_'): + continue + if k == k.capitalize(): + continue + v = getattr(module, k) + if not callable(v) or isinstance(v, typing.Type): + continue + if not hasattr(v, '__annotations__'): + continue + + vt = type(v) + if vt.__module__ == 'langchain_core.tools' and vt.__name__.endswith('Tool') and hasattr(v, 'func') and callable(v.func): + v = v.func + + print(f'INFO: Binding /{k}') + try: + app.post(k)(v) + except Exception as e: + print(f'WARNING: Failed to bind /{k}\n\t{e}') + +def main(files: typing.List[str], host: str = '0.0.0.0', port: int = 8000): + app = fastapi.FastAPI() + + for f in files: + if f.endswith('.py'): + sys.path.insert(0, str(Path(f).parent)) + + module = load_source_as_module(f) + else: + module = importlib.import_module(f) + + bind_functions(app, module) + + uvicorn.run(app, host=host, port=port) + +if __name__ == '__main__': + typer.run(main) + diff --git a/examples/agents/hermes_function_calling b/examples/agents/hermes_function_calling new file mode 160000 index 000000000..b4f757e27 --- /dev/null +++ b/examples/agents/hermes_function_calling @@ -0,0 +1 @@ +Subproject commit b4f757e27d87f4ab408f706f482c25a8e1508d59 diff --git a/examples/agents/requirements.txt b/examples/agents/requirements.txt new file mode 100644 index 000000000..2ff0ce927 --- /dev/null +++ b/examples/agents/requirements.txt @@ -0,0 +1,3 @@ +jsonargparse +pydantic +typer[all] \ No newline at end of file diff --git a/examples/agents/run_sandboxed_tools.sh b/examples/agents/run_sandboxed_tools.sh new file mode 100755 index 000000000..eb8eb252e --- /dev/null +++ b/examples/agents/run_sandboxed_tools.sh @@ -0,0 +1,82 @@ +#!/bin/bash +# +# Runs a Python script in a sandboxed environment and makes its functions available as a web service. +# +# git submodule add https://github.com/NousResearch/Hermes-Function-Calling examples/agents/hermes_function_calling +# python examples/agents/fastify.py examples/agents/hermes_function_calling/functions.py +# REQUIREMENTS_FILE=<( cat examples/agents/hermes_function_calling/requirements.txt | grep -vE "bitsandbytes|flash-attn" ) examples/agents/run_sandboxed_tools.sh examples/agents/hermes_function_calling/functions.py -e LOG_FOLDER=/data/inference_logs +set -euo pipefail + +script="$( realpath "$1" )" +script_folder="$(dirname "$script")" +shift 1 + +function cleanup { + rm -rf "$BUILD_DIR" + echo "Deleted $BUILD_DIR" +} +trap cleanup EXIT +BUILD_DIR=$(mktemp -d) +DATA_DIR="${DATA_DIR:-$HOME/.llama.cpp/sandbox}" +SCRIPT_DIR=$( cd "$(dirname "$0")" ; pwd ) + +REQUIREMENTS_FILE="${REQUIREMENTS_FILE:-}" +if [[ -z "$REQUIREMENTS_FILE" && -f "$script_folder/requirements.txt" ]]; then + REQUIREMENTS_FILE="$script_folder/requirements.txt" +fi +if [[ -n "$REQUIREMENTS_FILE" ]]; then + cp "$REQUIREMENTS_FILE" "$BUILD_DIR/script-requirements.txt" +else + touch $BUILD_DIR/script-requirements.txt +fi + +echo "INFO: using DATA_DIR: $DATA_DIR" + +cp \ + "$SCRIPT_DIR/fastify-requirements.txt" \ + "$SCRIPT_DIR/fastify.py" \ + "$BUILD_DIR" + +mkdir -p "$DATA_DIR" + +PORT=${PORT:-8088} + +# BASE_IMAGE=pytorch/pytorch:latest +# BASE_IMAGE=python:3.10-slim +BASE_IMAGE=python:3.11-slim +# torch +# FROM nvidia/cuda:12.1.1-runtime-ubuntu20.04 +# RUN apt-get update && \ +# apt-get install -y python3-pip python3-dev && \ +# rm -rf /var/lib/apt/lists/* + +echo " + FROM $BASE_IMAGE + RUN apt-get update + RUN apt-get install -y gcc python3-dev git cmake + RUN pip install --upgrade pip + RUN pip install packaging numpy + RUN mkdir /src /data + + # Copy resources in increasing likelihood of change, to keep as much as possible cached + COPY fastify-requirements.txt /root + RUN pip install -r /root/fastify-requirements.txt + COPY script-requirements.txt /root + RUN pip install -r /root/script-requirements.txt + COPY fastify.py /root + + WORKDIR /data + # ENTRYPOINT uvicorn fastify:app --reload + ENTRYPOINT PYTHONPATH=/src python /root/fastify.py --port=$PORT '/src/$( basename "$script" )' +" | docker build "$BUILD_DIR" -f - -t llama.cpp/tools-base + +echo "#" +echo "# Binding $script to http://localhost:$PORT/" +echo "#" +set -x +docker run \ + "$@" \ + --mount "type=bind,source=$( realpath "$script_folder" ),target=/src,readonly" \ + --mount "type=bind,source=$( realpath "$DATA_DIR" ),target=/data" \ + -p "$PORT:$PORT" \ + -it llama.cpp/tools-base \ No newline at end of file