agents: scripts to run scripts as sandboxed fastapi servers

.gitmodules

@@ -1,3 +1,6 @@
 [submodule "kompute"]
 	path = kompute
 	url = https://github.com/nomic-ai/kompute.git
+[submodule "examples/agents/hermes_function_calling"]
+	path = examples/agents/hermes_function_calling
+	url = https://github.com/NousResearch/Hermes-Function-Calling

examples/agents/README.md

@@ -0,0 +1,15 @@
+
+Edit `examples/agents/hermes_function_calling/utils.py`:
+
+```py
+log_folder = os.environ.get('LOG_FOLDER', os.path.join(script_dir, "inference_logs"))
+```
+
+Then run:
+
+```bash
+REQUIREMENTS_FILE=<( cat examples/agents/hermes_function_calling/requirements.txt | grep -vE "bitsandbytes|flash-attn" ) \
+  examples/agents/run_sandboxed_tools.sh \
+    examples/agents/hermes_function_calling/functions.py \
+    -e LOG_FOLDER=/data/inference_logs
+```

examples/agents/fastify-requirements.txt

@@ -0,0 +1,5 @@
+fastapi[all]
+pydantic
+sse-starlette
+uvicorn[all]
+typer[all]

examples/agents/fastify.py

@@ -0,0 +1,63 @@
+'''
+    Binds the functions of a python script as a FastAPI server.
+
+    This is useful in combination w/ the examples/agent/run_sandboxed_tools.sh
+'''
+import os, sys, typing, importlib.util
+from anyio import Path
+import fastapi, uvicorn
+import typer
+
+# from langchain_core.tools import BaseTool
+
+def load_source_as_module(source):
+    i = 0
+    while (module_name := f'mod_{i}') in sys.modules:
+        i += 1
+
+    spec = importlib.util.spec_from_file_location(module_name, source)
+    module = importlib.util.module_from_spec(spec)
+    sys.modules[module_name] = module
+    spec.loader.exec_module(module)
+    return module
+
+def bind_functions(app, module):
+    for k in dir(module):
+        if k.startswith('_'):
+            continue
+        if k == k.capitalize():
+            continue
+        v = getattr(module, k)
+        if not callable(v) or isinstance(v, typing.Type):
+            continue
+        if not hasattr(v, '__annotations__'):
+            continue
+
+        vt = type(v)
+        if vt.__module__ == 'langchain_core.tools' and vt.__name__.endswith('Tool') and hasattr(v, 'func') and callable(v.func):
+            v = v.func
+
+        print(f'INFO:     Binding /{k}')
+        try:
+            app.post(k)(v)
+        except Exception as e:
+            print(f'WARNING:    Failed to bind /{k}\n\t{e}')
+
+def main(files: typing.List[str], host: str = '0.0.0.0', port: int = 8000):
+    app = fastapi.FastAPI()
+
+    for f in files:
+        if f.endswith('.py'):
+            sys.path.insert(0, str(Path(f).parent))
+
+            module = load_source_as_module(f)
+        else:
+            module = importlib.import_module(f)
+
+        bind_functions(app, module)
+
+    uvicorn.run(app, host=host, port=port)
+
+if __name__ == '__main__':
+    typer.run(main)
+    

examples/agents/hermes_function_calling

@@ -0,0 +1 @@
+Subproject commit b4f757e27d87f4ab408f706f482c25a8e1508d59

examples/agents/requirements.txt

@@ -0,0 +1,3 @@
+jsonargparse
+pydantic
+typer[all]

examples/agents/run_sandboxed_tools.sh

@@ -0,0 +1,82 @@
+#!/bin/bash
+#
+# Runs a Python script in a sandboxed environment and makes its functions available as a web service.
+#
+# git submodule add https://github.com/NousResearch/Hermes-Function-Calling examples/agents/hermes_function_calling
+# python examples/agents/fastify.py examples/agents/hermes_function_calling/functions.py
+# REQUIREMENTS_FILE=<( cat examples/agents/hermes_function_calling/requirements.txt | grep -vE "bitsandbytes|flash-attn" ) examples/agents/run_sandboxed_tools.sh examples/agents/hermes_function_calling/functions.py -e LOG_FOLDER=/data/inference_logs
+set -euo pipefail
+
+script="$( realpath "$1" )"
+script_folder="$(dirname "$script")"
+shift 1
+
+function cleanup {      
+  rm -rf "$BUILD_DIR"
+  echo "Deleted $BUILD_DIR"
+}
+trap cleanup EXIT
+BUILD_DIR=$(mktemp -d)
+DATA_DIR="${DATA_DIR:-$HOME/.llama.cpp/sandbox}"
+SCRIPT_DIR=$( cd "$(dirname "$0")" ; pwd )
+
+REQUIREMENTS_FILE="${REQUIREMENTS_FILE:-}"
+if [[ -z "$REQUIREMENTS_FILE" && -f "$script_folder/requirements.txt" ]]; then
+    REQUIREMENTS_FILE="$script_folder/requirements.txt"
+fi
+if [[ -n "$REQUIREMENTS_FILE" ]]; then
+    cp "$REQUIREMENTS_FILE" "$BUILD_DIR/script-requirements.txt"
+else
+    touch $BUILD_DIR/script-requirements.txt
+fi
+
+echo "INFO: using DATA_DIR: $DATA_DIR"
+
+cp \
+    "$SCRIPT_DIR/fastify-requirements.txt" \
+    "$SCRIPT_DIR/fastify.py" \
+    "$BUILD_DIR"
+
+mkdir -p "$DATA_DIR"
+
+PORT=${PORT:-8088}
+
+# BASE_IMAGE=pytorch/pytorch:latest
+# BASE_IMAGE=python:3.10-slim
+BASE_IMAGE=python:3.11-slim
+# torch 
+# FROM nvidia/cuda:12.1.1-runtime-ubuntu20.04 
+# RUN apt-get update && \
+#     apt-get install -y python3-pip python3-dev && \
+#     rm -rf /var/lib/apt/lists/*
+
+echo "
+    FROM     $BASE_IMAGE
+    RUN      apt-get update
+    RUN      apt-get install -y gcc python3-dev git cmake
+    RUN      pip install --upgrade pip
+    RUN      pip install packaging numpy
+    RUN      mkdir /src /data
+
+    # Copy resources in increasing likelihood of change, to keep as much as possible cached
+    COPY     fastify-requirements.txt /root
+    RUN      pip install -r /root/fastify-requirements.txt
+    COPY     script-requirements.txt  /root
+    RUN      pip install -r /root/script-requirements.txt
+    COPY     fastify.py               /root
+
+    WORKDIR  /data
+    # ENTRYPOINT uvicorn fastify:app --reload
+    ENTRYPOINT PYTHONPATH=/src python /root/fastify.py --port=$PORT '/src/$( basename "$script" )'
+" | docker build "$BUILD_DIR" -f - -t llama.cpp/tools-base
+
+echo "#"
+echo "# Binding $script to http://localhost:$PORT/"
+echo "#"
+set -x
+docker run \
+    "$@" \
+    --mount "type=bind,source=$( realpath "$script_folder" ),target=/src,readonly" \
+    --mount "type=bind,source=$( realpath "$DATA_DIR" ),target=/data" \
+    -p "$PORT:$PORT" \
+    -it llama.cpp/tools-base