add python code interpreter

2025-02-08 13:01:02 +01:00 · 2025-02-08 13:01:02 +01:00 · 483a3bc2ad
commit 483a3bc2ad
parent 422e53e607
12 changed files with 406 additions and 85 deletions
--- a/examples/server/webui/src/App.tsx
+++ b/examples/server/webui/src/App.tsx
@ -1,8 +1,9 @@
 import { HashRouter, Outlet, Route, Routes } from 'react-router';
 import Header from './components/Header';
 import Sidebar from './components/Sidebar';
-import { AppContextProvider } from './utils/app.context';
+import { AppContextProvider, useAppContext } from './utils/app.context';
 import ChatScreen from './components/ChatScreen';
+import SettingDialog from './components/SettingDialog';

 function App() {
  return (
@ -22,13 +23,18 @@ function App() {
 }

 function AppLayout() {
+  const { showSettings, setShowSettings } = useAppContext();
  return (
    <>
      <Sidebar />
-      <div className="chat-screen drawer-content grow flex flex-col h-screen w-screen mx-auto px-4">
+      <div className="drawer-content grow flex flex-col h-screen w-screen mx-auto px-4 overflow-auto">
        <Header />
        <Outlet />
      </div>
+      {<SettingDialog
+        show={showSettings}
+        onClose={() => setShowSettings(false)}
+      />}
    </>
  );
 }
--- a/examples/server/webui/src/Config.ts
+++ b/examples/server/webui/src/Config.ts
@ -10,6 +10,7 @@ export const BASE_URL = new URL('.', document.baseURI).href

 export const CONFIG_DEFAULT = {
  // Note: in order not to introduce breaking changes, please keep the same data type (number, string, etc) if you want to change the default value. Do not use null or undefined for default value.
+  // Do not use nested objects, keep it single level. Prefix the key if you need to group them.
  apiKey: '',
  systemMessage: 'You are a helpful assistant.',
  showTokensPerSecond: false,
@ -36,6 +37,8 @@ export const CONFIG_DEFAULT = {
  dry_penalty_last_n: -1,
  max_tokens: -1,
  custom: '', // custom json-stringified object
+  // experimental features
+  pyIntepreterEnabled: false,
 };
 export const CONFIG_INFO: Record<string, string> = {
  apiKey: 'Set the API Key if you are using --api-key option for the server.',
--- a/examples/server/webui/src/components/CanvasPyInterpreter.tsx
+++ b/examples/server/webui/src/components/CanvasPyInterpreter.tsx
@ -0,0 +1,113 @@
+import { useEffect, useState } from 'react';
+import { useAppContext } from '../utils/app.context';
+import { XCloseButton } from '../utils/common';
+import { delay } from '../utils/misc';
+import StorageUtils from '../utils/storage';
+import { CanvasType } from '../utils/types';
+import { PlayIcon } from '@heroicons/react/24/outline';
+
+const PyodideWrapper = {
+  load: async function () {
+    // load pyodide from CDN
+    // @ts-expect-error experimental pyodide
+    if (window.addedScriptPyodide) return;
+    // @ts-expect-error experimental pyodide
+    window.addedScriptPyodide = true;
+    const scriptElem = document.createElement('script');
+    scriptElem.src = 'https://cdn.jsdelivr.net/pyodide/v0.27.2/full/pyodide.js';
+    document.body.appendChild(scriptElem);
+  },
+
+  run: async function (code: string) {
+    PyodideWrapper.load();
+
+    // wait for pyodide to be loaded
+    // @ts-expect-error experimental pyodide
+    while (!window.loadPyodide) {
+      await delay(100);
+    }
+    const stdOutAndErr: string[] = [];
+    // @ts-expect-error experimental pyodide
+    const pyodide = await window.loadPyodide({
+      stdout: (data: string) => stdOutAndErr.push(data),
+      stderr: (data: string) => stdOutAndErr.push(data),
+    });
+    const result = await pyodide.runPythonAsync(code);
+    if (result) {
+      stdOutAndErr.push(result.toString());
+    }
+    return stdOutAndErr.join('');
+  },
+};
+
+if (StorageUtils.getConfig().pyIntepreterEnabled) {
+  PyodideWrapper.load();
+}
+
+export default function CanvasPyInterpreter() {
+  const { canvasData, setCanvasData } = useAppContext();
+
+  const [running, setRunning] = useState(false);
+  const [output, setOutput] = useState('');
+
+  const runCode = async () => {
+    const code = canvasData?.content;
+    if (!code) return;
+    setRunning(true);
+    setOutput('Running...');
+    const out = await PyodideWrapper.run(code);
+    setOutput(out);
+    setRunning(false);
+  };
+
+  // run code on mount
+  useEffect(() => {
+    runCode();
+    // eslint-disable-next-line react-hooks/exhaustive-deps
+  }, []);
+
+  if (canvasData?.type !== CanvasType.PY_INTERPRETER) {
+    return null;
+  }
+
+  return (
+    <div className="card bg-base-200 w-full h-full shadow-xl">
+      <div className="card-body">
+        <div className="flex justify-between items-center mb-4">
+          <span className="text-lg font-bold">Pyodide</span>
+          <XCloseButton
+            className="bg-base-100"
+            onClick={() => setCanvasData(null)}
+          />
+        </div>
+        <div className="grid grid-rows-3 gap-4 h-full">
+          <textarea
+            className="textarea textarea-bordered w-full h-full font-mono"
+            value={canvasData.content}
+            onChange={(e) =>
+              setCanvasData({
+                ...canvasData,
+                content: e.target.value,
+              })
+            }
+          ></textarea>
+          <div className="font-mono flex flex-col row-span-2">
+            <div className="flex items-center mb-2">
+              <button
+                className="btn btn-sm bg-base-100"
+                onClick={runCode}
+                disabled={running}
+              >
+                <PlayIcon className="h-6 w-6" />{' '}
+                {running ? 'Running...' : 'Run'}
+              </button>
+            </div>
+            <pre className="bg-slate-900 rounded-md grow text-gray-200 p-3">
+              {output}
+            </pre>
+          </div>
+        </div>
+      </div>
+    </div>
+  );
+}
--- a/examples/server/webui/src/components/ChatMessage.tsx
+++ b/examples/server/webui/src/components/ChatMessage.tsx
@ -149,11 +149,17 @@ export default function ChatMessage({
                          )}
                        </summary>
                        <div className="collapse-content">
-                          <MarkdownDisplay content={thought} />
+                          <MarkdownDisplay
+                            content={thought}
+                            isGenerating={isPending}
+                          />
                        </div>
                      </details>
                    )}
-                    <MarkdownDisplay content={content} />
+                    <MarkdownDisplay
+                      content={content}
+                      isGenerating={isPending}
+                    />
                  </div>
                </>
              )}
--- a/examples/server/webui/src/components/ChatScreen.tsx
+++ b/examples/server/webui/src/components/ChatScreen.tsx
@ -4,6 +4,8 @@ import StorageUtils from '../utils/storage';
 import { useNavigate } from 'react-router';
 import ChatMessage from './ChatMessage';
 import { PendingMessage } from '../utils/types';
+import { classNames } from '../utils/misc';
+import CanvasPyInterpreter from './CanvasPyInterpreter';

 export default function ChatScreen() {
  const {
@ -12,6 +14,7 @@ export default function ChatScreen() {
    isGenerating,
    stopGenerating,
    pendingMessages,
+    canvasData,
  } = useAppContext();
  const [inputMsg, setInputMsg] = useState('');
  const containerRef = useRef<HTMLDivElement>(null);
@ -59,65 +62,82 @@ export default function ChatScreen() {
  };

  return (
-    <>
-      {/* chat messages */}
-      <div
-        id="messages-list"
-        className="flex flex-col grow overflow-y-auto"
-        ref={containerRef}
-      >
-        <div className="mt-auto flex justify-center">
-          {/* placeholder to shift the message to the bottom */}
-          {viewingConversation ? '' : 'Send a message to start'}
+    <div
+      className={classNames({
+        'grid gap-8 grow': true,
+        'grid-cols-2': !!canvasData,
+        'grid-cols-1': !canvasData,
+      })}
+    >
+      <div className="flex flex-col w-full max-w-[900px] mx-auto">
+        {/* chat messages */}
+        <div
+          id="messages-list"
+          className="flex flex-col grow overflow-y-auto"
+          ref={containerRef}
+        >
+          <div className="mt-auto flex justify-center">
+            {/* placeholder to shift the message to the bottom */}
+            {viewingConversation ? '' : 'Send a message to start'}
+          </div>
+          {viewingConversation?.messages.map((msg) => (
+            <ChatMessage
+              key={msg.id}
+              msg={msg}
+              scrollToBottom={scrollToBottom}
+            />
+          ))}
+
+          {pendingMsg && (
+            <ChatMessage
+              msg={pendingMsg}
+              scrollToBottom={scrollToBottom}
+              isPending
+              id="pending-msg"
+            />
+          )}
        </div>
-        {viewingConversation?.messages.map((msg) => (
-          <ChatMessage key={msg.id} msg={msg} scrollToBottom={scrollToBottom} />
-        ))}

-        {pendingMsg && (
-          <ChatMessage
-            msg={pendingMsg}
-            scrollToBottom={scrollToBottom}
-            isPending
-            id="pending-msg"
-          />
-        )}
+        {/* chat input */}
+        <div className="flex flex-row items-center pt-8 pb-6 sticky bottom-0 bg-base-100">
+          <textarea
+            className="textarea textarea-bordered w-full"
+            placeholder="Type a message (Shift+Enter to add a new line)"
+            value={inputMsg}
+            onChange={(e) => setInputMsg(e.target.value)}
+            onKeyDown={(e) => {
+              if (e.key === 'Enter' && e.shiftKey) return;
+              if (e.key === 'Enter' && !e.shiftKey) {
+                e.preventDefault();
+                sendNewMessage();
+              }
+            }}
+            id="msg-input"
+            dir="auto"
+          ></textarea>
+          {isGenerating(currConvId) ? (
+            <button
+              className="btn btn-neutral ml-2"
+              onClick={() => stopGenerating(currConvId)}
+            >
+              Stop
+            </button>
+          ) : (
+            <button
+              className="btn btn-primary ml-2"
+              onClick={sendNewMessage}
+              disabled={inputMsg.trim().length === 0}
+            >
+              Send
+            </button>
+          )}
+        </div>
      </div>
-
-      {/* chat input */}
-      <div className="flex flex-row items-center mt-8 mb-6">
-        <textarea
-          className="textarea textarea-bordered w-full"
-          placeholder="Type a message (Shift+Enter to add a new line)"
-          value={inputMsg}
-          onChange={(e) => setInputMsg(e.target.value)}
-          onKeyDown={(e) => {
-            if (e.key === 'Enter' && e.shiftKey) return;
-            if (e.key === 'Enter' && !e.shiftKey) {
-              e.preventDefault();
-              sendNewMessage();
-            }
-          }}
-          id="msg-input"
-          dir="auto"
-        ></textarea>
-        {isGenerating(currConvId) ? (
-          <button
-            className="btn btn-neutral ml-2"
-            onClick={() => stopGenerating(currConvId)}
-          >
-            Stop
-          </button>
-        ) : (
-          <button
-            className="btn btn-primary ml-2"
-            onClick={sendNewMessage}
-            disabled={inputMsg.trim().length === 0}
-          >
-            Send
-          </button>
-        )}
-      </div>
-    </>
+      {canvasData && (
+        <div className="w-full sticky top-[8em] h-[calc(100vh-9em)]">
+          <CanvasPyInterpreter />
+        </div>
+      )}
+    </div>
  );
 }
--- a/examples/server/webui/src/components/Header.tsx
+++ b/examples/server/webui/src/components/Header.tsx
@ -5,12 +5,11 @@ import { classNames } from '../utils/misc';
 import daisyuiThemes from 'daisyui/src/theming/themes';
 import { THEMES } from '../Config';
 import { useNavigate } from 'react-router';
-import SettingDialog from './SettingDialog';

 export default function Header() {
  const navigate = useNavigate();
  const [selectedTheme, setSelectedTheme] = useState(StorageUtils.getTheme());
-  const [showSettingDialog, setShowSettingDialog] = useState(false);
+  const { setShowSettings } = useAppContext();

  const setTheme = (theme: string) => {
    StorageUtils.setTheme(theme);
@ -54,7 +53,7 @@ export default function Header() {
  };

  return (
-    <div className="flex flex-row items-center mt-6 mb-6">
+    <div className="flex flex-row items-center pt-6 pb-6 sticky top-0 z-10 bg-base-100">
      {/* open sidebar button */}
      <label htmlFor="toggle-drawer" className="btn btn-ghost lg:hidden">
        <svg
@ -109,7 +108,7 @@ export default function Header() {
          </ul>
        </div>
        <div className="tooltip tooltip-bottom" data-tip="Settings">
-          <button className="btn" onClick={() => setShowSettingDialog(true)}>
+          <button className="btn" onClick={() => setShowSettings(true)}>
            {/* settings button */}
            <svg
              xmlns="http://www.w3.org/2000/svg"
@ -172,11 +171,6 @@ export default function Header() {
          </div>
        </div>
      </div>
-
-      <SettingDialog
-        show={showSettingDialog}
-        onClose={() => setShowSettingDialog(false)}
-      />
    </div>
  );
 }
--- a/examples/server/webui/src/components/MarkdownDisplay.tsx
+++ b/examples/server/webui/src/components/MarkdownDisplay.tsx
@ -9,8 +9,16 @@ import 'katex/dist/katex.min.css';
 import { classNames, copyStr } from '../utils/misc';
 import { ElementContent, Root } from 'hast';
 import { visit } from 'unist-util-visit';
+import { useAppContext } from '../utils/app.context';
+import { CanvasType } from '../utils/types';

-export default function MarkdownDisplay({ content }: { content: string }) {
+export default function MarkdownDisplay({
+  content,
+  isGenerating,
+}: {
+  content: string;
+  isGenerating?: boolean;
+}) {
  const preprocessedContent = useMemo(
    () => preprocessLaTeX(content),
    [content]
@ -21,7 +29,11 @@ export default function MarkdownDisplay({ content }: { content: string }) {
      rehypePlugins={[rehypeHightlight, rehypeKatex, rehypeCustomCopyButton]}
      components={{
        button: (props) => (
-          <CopyCodeButton {...props} origContent={preprocessedContent} />
+          <CodeBlockButtons
+            {...props}
+            isGenerating={isGenerating}
+            origContent={preprocessedContent}
+          />
        ),
        // note: do not use "pre", "p" or other basic html elements here, it will cause the node to re-render when the message is being generated (this should be a bug with react-markdown, not sure how to fix it)
      }}
@ -31,11 +43,12 @@ export default function MarkdownDisplay({ content }: { content: string }) {
  );
 }

-const CopyCodeButton: React.ElementType<
+const CodeBlockButtons: React.ElementType<
  React.ClassAttributes<HTMLButtonElement> &
    React.HTMLAttributes<HTMLButtonElement> &
-    ExtraProps & { origContent: string }
-> = ({ node, origContent }) => {
+    ExtraProps & { origContent: string; isGenerating?: boolean }
+> = ({ node, origContent, isGenerating }) => {
+  const { config } = useAppContext();
  const startOffset = node?.position?.start.offset ?? 0;
  const endOffset = node?.position?.end.offset ?? 0;

@ -48,6 +61,19 @@ const CopyCodeButton: React.ElementType<
    [origContent, startOffset, endOffset]
  );

+  const codeLanguage = useMemo(
+    () =>
+      origContent
+        .substring(startOffset, startOffset + 10)
+        .match(/^```([^\n]+)\n/)?.[1] ?? '',
+    [origContent, startOffset]
+  );
+
+  const canRunCode =
+    !isGenerating &&
+    config.pyIntepreterEnabled &&
+    codeLanguage.startsWith('py');
+
  return (
    <div
      className={classNames({
@ -56,6 +82,12 @@ const CopyCodeButton: React.ElementType<
      })}
    >
      <CopyButton className="badge btn-mini" content={copiedContent} />
+      {canRunCode && (
+        <RunPyCodeButton
+          className="badge btn-mini ml-2"
+          content={copiedContent}
+        />
+      )}
    </div>
  );
 };
@ -82,6 +114,31 @@ export const CopyButton = ({
  );
 };

+export const RunPyCodeButton = ({
+  content,
+  className,
+}: {
+  content: string;
+  className?: string;
+}) => {
+  const { setCanvasData } = useAppContext();
+  return (
+    <>
+      <button
+        className={className}
+        onClick={() =>
+          setCanvasData({
+            type: CanvasType.PY_INTERPRETER,
+            content,
+          })
+        }
+      >
+        ▶️ Run
+      </button>
+    </>
+  );
+};
+
 /**
 * This injects the "button" element before each "pre" element.
 * The actual button will be replaced with a react component in the MarkdownDisplay.
--- a/examples/server/webui/src/components/SettingDialog.tsx
+++ b/examples/server/webui/src/components/SettingDialog.tsx
@ -5,12 +5,14 @@ import { isDev } from '../Config';
 import StorageUtils from '../utils/storage';
 import { classNames, isBoolean, isNumeric, isString } from '../utils/misc';
 import {
+  BeakerIcon,
  ChatBubbleOvalLeftEllipsisIcon,
  Cog6ToothIcon,
  FunnelIcon,
  HandRaisedIcon,
  SquaresPlusIcon,
 } from '@heroicons/react/24/outline';
+import { OpenInNewTab } from '../utils/common';

 type SettKey = keyof typeof CONFIG_DEFAULT;

@ -194,14 +196,9 @@ const SETTING_SECTIONS: SettingSection[] = [
        label: (
          <>
            Custom JSON config (For more info, refer to{' '}
-            <a
-              className="underline"
-              href="https://github.com/ggerganov/llama.cpp/blob/master/examples/server/README.md"
-              target="_blank"
-              rel="noopener noreferrer"
-            >
+            <OpenInNewTab href="https://github.com/ggerganov/llama.cpp/blob/master/examples/server/README.md">
              server documentation
-            </a>
+            </OpenInNewTab>
            )
          </>
        ),
@ -209,6 +206,56 @@ const SETTING_SECTIONS: SettingSection[] = [
      },
    ],
  },
+  {
+    title: (
+      <>
+        <BeakerIcon className={ICON_CLASSNAME} />
+        Experimental
+      </>
+    ),
+    fields: [
+      {
+        type: SettingInputType.CUSTOM,
+        key: 'custom', // dummy key, won't be used
+        component: () => (
+          <>
+            <p className="mb-8">
+              Experimental features are not guaranteed to work correctly.
+              <br />
+              <br />
+              If you encounter any problems, create a{' '}
+              <OpenInNewTab href="https://github.com/ggerganov/llama.cpp/issues/new?template=019-bug-misc.yml">
+                Bug (misc.)
+              </OpenInNewTab>{' '}
+              report on Github. Please also specify <b>webui/experimental</b> on
+              the report title and include screenshots.
+              <br />
+              <br />
+              Some features may require packages downloaded from CDN, so they
+              need internet connection.
+            </p>
+          </>
+        ),
+      },
+      {
+        type: SettingInputType.CHECKBOX,
+        label: (
+          <>
+            <b>Enable Python interpreter</b>
+            <br />
+            <small className="text-xs">
+              This feature uses{' '}
+              <OpenInNewTab href="https://pyodide.org">pyodide</OpenInNewTab>,
+              downloaded from CDN. To use this feature, ask the LLM to generate
+              python code inside a markdown code block. You will see a "Run"
+              button on the code block, near the "Copy" button.
+            </small>
+          </>
+        ),
+        key: 'pyIntepreterEnabled',
+      },
+    ],
+  },
 ];

 export default function SettingDialog({
@ -278,7 +325,7 @@ export default function SettingDialog({
  };

  return (
-    <dialog className={`modal ${show ? 'modal-open' : ''}`}>
+    <dialog className={classNames({ 'modal': true, 'modal-open': show })}>
      <div className="modal-box w-11/12 max-w-3xl">
        <h3 className="text-lg font-bold mb-6">Settings</h3>
        <div className="flex flex-col md:flex-row h-[calc(90vh-12rem)]">
@ -479,7 +526,7 @@ function SettingsModalCheckbox({
    <div className="flex flex-row items-center mb-2">
      <input
        type="checkbox"
-        className="checkbox"
+        className="toggle"
        checked={value}
        onChange={(e) => onChange(e.target.checked)}
      />
--- a/examples/server/webui/src/utils/app.context.tsx
+++ b/examples/server/webui/src/utils/app.context.tsx
@ -1,5 +1,11 @@
 import React, { createContext, useContext, useEffect, useState } from 'react';
-import { APIMessage, Conversation, Message, PendingMessage } from './types';
+import {
+  APIMessage,
+  CanvasData,
+  Conversation,
+  Message,
+  PendingMessage,
+} from './types';
 import StorageUtils from './storage';
 import {
  filterThoughtFromMsgs,
@ -10,6 +16,7 @@ import { BASE_URL, CONFIG_DEFAULT, isDev } from '../Config';
 import { matchPath, useLocation } from 'react-router';

 interface AppContextValue {
+  // conversations and messages
  viewingConversation: Conversation | null;
  pendingMessages: Record<Conversation['id'], PendingMessage>;
  isGenerating: (convId: string) => boolean;
@ -26,8 +33,15 @@ interface AppContextValue {
    onChunk?: CallbackGeneratedChunk
  ) => Promise<void>;

+  // canvas
+  canvasData: CanvasData | null;
+  setCanvasData: (data: CanvasData | null) => void;
+
+  // config
  config: typeof CONFIG_DEFAULT;
  saveConfig: (config: typeof CONFIG_DEFAULT) => void;
+  showSettings: boolean;
+  setShowSettings: (show: boolean) => void;
 }

 // for now, this callback is only used for scrolling to the bottom of the chat
@ -54,8 +68,13 @@ export const AppContextProvider = ({
    Record<Conversation['id'], AbortController>
  >({});
  const [config, setConfig] = useState(StorageUtils.getConfig());
+  const [canvasData, setCanvasData] = useState<CanvasData | null>(null);
+  const [showSettings, setShowSettings] = useState(false);

+  // handle change when the convId from URL is changed
  useEffect(() => {
+    // also reset the canvas data
+    setCanvasData(null);
    const handleConversationChange = (changedConvId: string) => {
      if (changedConvId !== convId) return;
      setViewingConversation(StorageUtils.getOneConversation(convId));
@ -292,8 +311,12 @@ export const AppContextProvider = ({
        sendMessage,
        stopGenerating,
        replaceMessageAndGenerate,
+        canvasData,
+        setCanvasData,
        config,
        saveConfig,
+        showSettings,
+        setShowSettings,
      }}
    >
      {children}
--- a/examples/server/webui/src/utils/common.tsx
+++ b/examples/server/webui/src/utils/common.tsx
@ -0,0 +1,38 @@
+export const XCloseButton: React.ElementType<
+  React.ClassAttributes<HTMLButtonElement> &
+    React.HTMLAttributes<HTMLButtonElement>
+> = ({ className, ...props }) => (
+  <button className={`btn btn-square btn-sm ${className ?? ''}`} {...props}>
+    <svg
+      xmlns="http://www.w3.org/2000/svg"
+      className="h-6 w-6"
+      fill="none"
+      viewBox="0 0 24 24"
+      stroke="currentColor"
+    >
+      <path
+        strokeLinecap="round"
+        strokeLinejoin="round"
+        strokeWidth="2"
+        d="M6 18L18 6M6 6l12 12"
+      />
+    </svg>
+  </button>
+);
+
+export const OpenInNewTab = ({
+  href,
+  children,
+}: {
+  href: string;
+  children: string;
+}) => (
+  <a
+    className="underline"
+    href={href}
+    target="_blank"
+    rel="noopener noreferrer"
+  >
+    {children}
+  </a>
+);
--- a/examples/server/webui/src/utils/misc.ts
+++ b/examples/server/webui/src/utils/misc.ts
@ -85,3 +85,6 @@ export function classNames(classes: Record<string, boolean>): string {
    .map(([key, _]) => key)
    .join(' ');
 }
+
+export const delay = (ms: number) =>
+  new Promise((resolve) => setTimeout(resolve, ms));
--- a/examples/server/webui/src/utils/types.ts
+++ b/examples/server/webui/src/utils/types.ts
@ -23,3 +23,14 @@ export interface Conversation {
 export type PendingMessage = Omit<Message, 'content'> & {
  content: string | null;
 };
+
+export enum CanvasType {
+  PY_INTERPRETER,
+}
+
+export interface CanvasPyInterpreter {
+  type: CanvasType.PY_INTERPRETER;
+  content: string;
+}
+
+export type CanvasData = CanvasPyInterpreter;