diff --git a/examples/server/public/index.html.gz b/examples/server/public/index.html.gz index 662a30f00..141e80920 100644 Binary files a/examples/server/public/index.html.gz and b/examples/server/public/index.html.gz differ diff --git a/examples/server/server.cpp b/examples/server/server.cpp index 0aff9b86b..0718806c8 100644 --- a/examples/server/server.cpp +++ b/examples/server/server.cpp @@ -4378,6 +4378,9 @@ int main(int argc, char ** argv) { res.set_content("Error: gzip is not supported by this browser", "text/plain"); } else { res.set_header("Content-Encoding", "gzip"); + // COEP and COOP headers, required by pyodide (python interpreter) + res.set_header("Cross-Origin-Embedder-Policy", "require-corp"); + res.set_header("Cross-Origin-Opener-Policy", "same-origin"); res.set_content(reinterpret_cast(index_html_gz), index_html_gz_len, "text/html; charset=utf-8"); } return false; diff --git a/examples/server/webui/package-lock.json b/examples/server/webui/package-lock.json index e69fd2aa5..c6c5de3c0 100644 --- a/examples/server/webui/package-lock.json +++ b/examples/server/webui/package-lock.json @@ -8,6 +8,7 @@ "name": "webui", "version": "0.0.0", "dependencies": { + "@heroicons/react": "^2.2.0", "@sec-ant/readable-stream": "^0.6.0", "@vscode/markdown-it-katex": "^1.1.1", "autoprefixer": "^10.4.20", @@ -902,6 +903,15 @@ "node": "^18.18.0 || ^20.9.0 || >=21.1.0" } }, + "node_modules/@heroicons/react": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/@heroicons/react/-/react-2.2.0.tgz", + "integrity": "sha512-LMcepvRaS9LYHJGsF0zzmgKCUim/X3N/DQKc4jepAXJ7l8QxJ1PmxJzqplF2Z3FE4PqBAIGyJAQ/w4B5dsqbtQ==", + "license": "MIT", + "peerDependencies": { + "react": ">= 16 || ^19.0.0-rc" + } + }, "node_modules/@humanfs/core": { "version": "0.19.1", "resolved": "https://registry.npmjs.org/@humanfs/core/-/core-0.19.1.tgz", diff --git a/examples/server/webui/package.json b/examples/server/webui/package.json index f3c7dde43..3be2b14de 100644 --- a/examples/server/webui/package.json +++ b/examples/server/webui/package.json @@ -11,6 +11,7 @@ "preview": "vite preview" }, "dependencies": { + "@heroicons/react": "^2.2.0", "@sec-ant/readable-stream": "^0.6.0", "@vscode/markdown-it-katex": "^1.1.1", "autoprefixer": "^10.4.20", diff --git a/examples/server/webui/src/App.tsx b/examples/server/webui/src/App.tsx index d151ba291..2ce734682 100644 --- a/examples/server/webui/src/App.tsx +++ b/examples/server/webui/src/App.tsx @@ -1,8 +1,9 @@ import { HashRouter, Outlet, Route, Routes } from 'react-router'; import Header from './components/Header'; import Sidebar from './components/Sidebar'; -import { AppContextProvider } from './utils/app.context'; +import { AppContextProvider, useAppContext } from './utils/app.context'; import ChatScreen from './components/ChatScreen'; +import SettingDialog from './components/SettingDialog'; function App() { return ( @@ -22,13 +23,23 @@ function App() { } function AppLayout() { + const { showSettings, setShowSettings } = useAppContext(); return ( <> -
+
+ { + setShowSettings(false)} + /> + } ); } diff --git a/examples/server/webui/src/Config.ts b/examples/server/webui/src/Config.ts index 1860ffcc9..779ed9bf7 100644 --- a/examples/server/webui/src/Config.ts +++ b/examples/server/webui/src/Config.ts @@ -10,6 +10,7 @@ export const BASE_URL = new URL('.', document.baseURI).href export const CONFIG_DEFAULT = { // Note: in order not to introduce breaking changes, please keep the same data type (number, string, etc) if you want to change the default value. Do not use null or undefined for default value. + // Do not use nested objects, keep it single level. Prefix the key if you need to group them. apiKey: '', systemMessage: 'You are a helpful assistant.', showTokensPerSecond: false, @@ -36,6 +37,8 @@ export const CONFIG_DEFAULT = { dry_penalty_last_n: -1, max_tokens: -1, custom: '', // custom json-stringified object + // experimental features + pyIntepreterEnabled: false, }; export const CONFIG_INFO: Record = { apiKey: 'Set the API Key if you are using --api-key option for the server.', diff --git a/examples/server/webui/src/components/CanvasPyInterpreter.tsx b/examples/server/webui/src/components/CanvasPyInterpreter.tsx new file mode 100644 index 000000000..c2707fe20 --- /dev/null +++ b/examples/server/webui/src/components/CanvasPyInterpreter.tsx @@ -0,0 +1,195 @@ +import { useEffect, useState } from 'react'; +import { useAppContext } from '../utils/app.context'; +import { OpenInNewTab, XCloseButton } from '../utils/common'; +import { CanvasType } from '../utils/types'; +import { PlayIcon, StopIcon } from '@heroicons/react/24/outline'; +import StorageUtils from '../utils/storage'; + +const canInterrupt = typeof SharedArrayBuffer === 'function'; + +// adapted from https://pyodide.org/en/stable/usage/webworker.html +const WORKER_CODE = ` +importScripts("https://cdn.jsdelivr.net/pyodide/v0.27.2/full/pyodide.js"); + +let stdOutAndErr = []; + +let pyodideReadyPromise = loadPyodide({ + stdout: (data) => stdOutAndErr.push(data), + stderr: (data) => stdOutAndErr.push(data), +}); + +let alreadySetBuff = false; + +self.onmessage = async (event) => { + stdOutAndErr = []; + + // make sure loading is done + const pyodide = await pyodideReadyPromise; + const { id, python, context, interruptBuffer } = event.data; + + if (interruptBuffer && !alreadySetBuff) { + pyodide.setInterruptBuffer(interruptBuffer); + alreadySetBuff = true; + } + + // Now load any packages we need, run the code, and send the result back. + await pyodide.loadPackagesFromImports(python); + + // make a Python dictionary with the data from content + const dict = pyodide.globals.get("dict"); + const globals = dict(Object.entries(context)); + try { + self.postMessage({ id, running: true }); + // Execute the python code in this context + const result = pyodide.runPython(python, { globals }); + self.postMessage({ result, id, stdOutAndErr }); + } catch (error) { + self.postMessage({ error: error.message, id }); + } + interruptBuffer[0] = 0; +}; +`; + +let worker: Worker; +const interruptBuffer = canInterrupt + ? new Uint8Array(new SharedArrayBuffer(1)) + : null; + +const startWorker = () => { + if (!worker) { + worker = new Worker( + URL.createObjectURL(new Blob([WORKER_CODE], { type: 'text/javascript' })) + ); + } +}; + +if (StorageUtils.getConfig().pyIntepreterEnabled) { + startWorker(); +} + +const runCodeInWorker = ( + pyCode: string, + callbackRunning: () => void +): { + donePromise: Promise; + interrupt: () => void; +} => { + startWorker(); + const id = Math.random() * 1e8; + const context = {}; + if (interruptBuffer) { + interruptBuffer[0] = 0; + } + + const donePromise = new Promise((resolve) => { + worker.onmessage = (event) => { + const { error, stdOutAndErr, running } = event.data; + if (id !== event.data.id) return; + if (running) { + callbackRunning(); + return; + } else if (error) { + resolve(error.toString()); + } else { + resolve(stdOutAndErr.join('\n')); + } + }; + worker.postMessage({ id, python: pyCode, context, interruptBuffer }); + }); + + const interrupt = () => { + console.log('Interrupting...'); + console.trace(); + if (interruptBuffer) { + interruptBuffer[0] = 2; + } + }; + + return { donePromise, interrupt }; +}; + +export default function CanvasPyInterpreter() { + const { canvasData, setCanvasData } = useAppContext(); + + const [code, setCode] = useState(canvasData?.content ?? ''); // copy to avoid direct mutation + const [running, setRunning] = useState(false); + const [output, setOutput] = useState(''); + const [interruptFn, setInterruptFn] = useState<() => void>(); + const [showStopBtn, setShowStopBtn] = useState(false); + + const runCode = async (pycode: string) => { + interruptFn?.(); + setRunning(true); + setOutput('Loading Pyodide...'); + const { donePromise, interrupt } = runCodeInWorker(pycode, () => { + setOutput('Running...'); + setShowStopBtn(canInterrupt); + }); + setInterruptFn(() => interrupt); + const out = await donePromise; + setOutput(out); + setRunning(false); + setShowStopBtn(false); + }; + + // run code on mount + useEffect(() => { + setCode(canvasData?.content ?? ''); + runCode(canvasData?.content ?? ''); + // eslint-disable-next-line react-hooks/exhaustive-deps + }, [canvasData?.content]); + + if (canvasData?.type !== CanvasType.PY_INTERPRETER) { + return null; + } + + return ( +
+
+
+ Python Interpreter + setCanvasData(null)} + /> +
+
+ +
+
+ + {showStopBtn && ( + + )} + + + Report a bug + + +
+ +
+
+
+
+ ); +} diff --git a/examples/server/webui/src/components/ChatMessage.tsx b/examples/server/webui/src/components/ChatMessage.tsx index 2666f81c0..ec72196ba 100644 --- a/examples/server/webui/src/components/ChatMessage.tsx +++ b/examples/server/webui/src/components/ChatMessage.tsx @@ -149,11 +149,17 @@ export default function ChatMessage({ )}
- +
)} - +
)} diff --git a/examples/server/webui/src/components/ChatScreen.tsx b/examples/server/webui/src/components/ChatScreen.tsx index d679f4ebb..dbc683ed1 100644 --- a/examples/server/webui/src/components/ChatScreen.tsx +++ b/examples/server/webui/src/components/ChatScreen.tsx @@ -1,9 +1,11 @@ -import { useEffect, useRef, useState } from 'react'; +import { useEffect, useState } from 'react'; import { useAppContext } from '../utils/app.context'; import StorageUtils from '../utils/storage'; import { useNavigate } from 'react-router'; import ChatMessage from './ChatMessage'; -import { PendingMessage } from '../utils/types'; +import { CanvasType, PendingMessage } from '../utils/types'; +import { classNames } from '../utils/misc'; +import CanvasPyInterpreter from './CanvasPyInterpreter'; export default function ChatScreen() { const { @@ -12,24 +14,24 @@ export default function ChatScreen() { isGenerating, stopGenerating, pendingMessages, + canvasData, } = useAppContext(); const [inputMsg, setInputMsg] = useState(''); - const containerRef = useRef(null); const navigate = useNavigate(); const currConvId = viewingConversation?.id ?? ''; const pendingMsg: PendingMessage | undefined = pendingMessages[currConvId]; const scrollToBottom = (requiresNearBottom: boolean) => { - if (!containerRef.current) return; - const msgListElem = containerRef.current; + const mainScrollElem = document.getElementById('main-scroll'); + if (!mainScrollElem) return; const spaceToBottom = - msgListElem.scrollHeight - - msgListElem.scrollTop - - msgListElem.clientHeight; + mainScrollElem.scrollHeight - + mainScrollElem.scrollTop - + mainScrollElem.clientHeight; if (!requiresNearBottom || spaceToBottom < 50) { setTimeout( - () => msgListElem.scrollTo({ top: msgListElem.scrollHeight }), + () => mainScrollElem.scrollTo({ top: mainScrollElem.scrollHeight }), 1 ); } @@ -58,66 +60,87 @@ export default function ChatScreen() { } }; + const hasCanvas = !!canvasData; + return ( - <> - {/* chat messages */} +
-
- {/* placeholder to shift the message to the bottom */} - {viewingConversation ? '' : 'Send a message to start'} + {/* chat messages */} +
+
+ {/* placeholder to shift the message to the bottom */} + {viewingConversation ? '' : 'Send a message to start'} +
+ {viewingConversation?.messages.map((msg) => ( + + ))} + + {pendingMsg && ( + + )}
- {viewingConversation?.messages.map((msg) => ( - - ))} - {pendingMsg && ( - + {/* chat input */} +
+ + {isGenerating(currConvId) ? ( + + ) : ( + + )} +
+
+
+ {canvasData?.type === CanvasType.PY_INTERPRETER && ( + )}
- - {/* chat input */} -
- - {isGenerating(currConvId) ? ( - - ) : ( - - )} -
- +
); } diff --git a/examples/server/webui/src/components/Header.tsx b/examples/server/webui/src/components/Header.tsx index 015264abc..505350313 100644 --- a/examples/server/webui/src/components/Header.tsx +++ b/examples/server/webui/src/components/Header.tsx @@ -5,12 +5,11 @@ import { classNames } from '../utils/misc'; import daisyuiThemes from 'daisyui/src/theming/themes'; import { THEMES } from '../Config'; import { useNavigate } from 'react-router'; -import SettingDialog from './SettingDialog'; export default function Header() { const navigate = useNavigate(); const [selectedTheme, setSelectedTheme] = useState(StorageUtils.getTheme()); - const [showSettingDialog, setShowSettingDialog] = useState(false); + const { setShowSettings } = useAppContext(); const setTheme = (theme: string) => { StorageUtils.setTheme(theme); @@ -54,7 +53,7 @@ export default function Header() { }; return ( -
+
{/* open sidebar button */}
-
- - setShowSettingDialog(false)} - />
); } diff --git a/examples/server/webui/src/components/MarkdownDisplay.tsx b/examples/server/webui/src/components/MarkdownDisplay.tsx index 814920a74..5b7a72591 100644 --- a/examples/server/webui/src/components/MarkdownDisplay.tsx +++ b/examples/server/webui/src/components/MarkdownDisplay.tsx @@ -9,8 +9,16 @@ import 'katex/dist/katex.min.css'; import { classNames, copyStr } from '../utils/misc'; import { ElementContent, Root } from 'hast'; import { visit } from 'unist-util-visit'; +import { useAppContext } from '../utils/app.context'; +import { CanvasType } from '../utils/types'; -export default function MarkdownDisplay({ content }: { content: string }) { +export default function MarkdownDisplay({ + content, + isGenerating, +}: { + content: string; + isGenerating?: boolean; +}) { const preprocessedContent = useMemo( () => preprocessLaTeX(content), [content] @@ -21,7 +29,11 @@ export default function MarkdownDisplay({ content }: { content: string }) { rehypePlugins={[rehypeHightlight, rehypeKatex, rehypeCustomCopyButton]} components={{ button: (props) => ( - + ), // note: do not use "pre", "p" or other basic html elements here, it will cause the node to re-render when the message is being generated (this should be a bug with react-markdown, not sure how to fix it) }} @@ -31,11 +43,12 @@ export default function MarkdownDisplay({ content }: { content: string }) { ); } -const CopyCodeButton: React.ElementType< +const CodeBlockButtons: React.ElementType< React.ClassAttributes & React.HTMLAttributes & - ExtraProps & { origContent: string } -> = ({ node, origContent }) => { + ExtraProps & { origContent: string; isGenerating?: boolean } +> = ({ node, origContent, isGenerating }) => { + const { config } = useAppContext(); const startOffset = node?.position?.start.offset ?? 0; const endOffset = node?.position?.end.offset ?? 0; @@ -48,14 +61,33 @@ const CopyCodeButton: React.ElementType< [origContent, startOffset, endOffset] ); + const codeLanguage = useMemo( + () => + origContent + .substring(startOffset, startOffset + 10) + .match(/^```([^\n]+)\n/)?.[1] ?? '', + [origContent, startOffset] + ); + + const canRunCode = + !isGenerating && + config.pyIntepreterEnabled && + codeLanguage.startsWith('py'); + return (
+ {canRunCode && ( + + )}
); }; @@ -82,6 +114,31 @@ export const CopyButton = ({ ); }; +export const RunPyCodeButton = ({ + content, + className, +}: { + content: string; + className?: string; +}) => { + const { setCanvasData } = useAppContext(); + return ( + <> + + + ); +}; + /** * This injects the "button" element before each "pre" element. * The actual button will be replaced with a react component in the MarkdownDisplay. @@ -95,9 +152,7 @@ function rehypeCustomCopyButton() { // replace current node preNode.properties.visited = 'true'; node.tagName = 'div'; - node.properties = { - className: 'relative my-4', - }; + node.properties = {}; // add node for button const btnNode: ElementContent = { type: 'element', diff --git a/examples/server/webui/src/components/SettingDialog.tsx b/examples/server/webui/src/components/SettingDialog.tsx index 5565ab7bb..592b93fa3 100644 --- a/examples/server/webui/src/components/SettingDialog.tsx +++ b/examples/server/webui/src/components/SettingDialog.tsx @@ -3,18 +3,27 @@ import { useAppContext } from '../utils/app.context'; import { CONFIG_DEFAULT, CONFIG_INFO } from '../Config'; import { isDev } from '../Config'; import StorageUtils from '../utils/storage'; -import { isBoolean, isNumeric, isString } from '../utils/misc'; +import { classNames, isBoolean, isNumeric, isString } from '../utils/misc'; +import { + BeakerIcon, + ChatBubbleOvalLeftEllipsisIcon, + Cog6ToothIcon, + FunnelIcon, + HandRaisedIcon, + SquaresPlusIcon, +} from '@heroicons/react/24/outline'; +import { OpenInNewTab } from '../utils/common'; type SettKey = keyof typeof CONFIG_DEFAULT; -const COMMON_SAMPLER_KEYS: SettKey[] = [ +const BASIC_KEYS: SettKey[] = [ 'temperature', 'top_k', 'top_p', 'min_p', 'max_tokens', ]; -const OTHER_SAMPLER_KEYS: SettKey[] = [ +const SAMPLER_KEYS: SettKey[] = [ 'dynatemp_range', 'dynatemp_exponent', 'typical_p', @@ -32,6 +41,223 @@ const PENALTY_KEYS: SettKey[] = [ 'dry_penalty_last_n', ]; +enum SettingInputType { + SHORT_INPUT, + LONG_INPUT, + CHECKBOX, + CUSTOM, +} + +interface SettingFieldInput { + type: Exclude; + label: string | React.ReactElement; + help?: string | React.ReactElement; + key: SettKey; +} + +interface SettingFieldCustom { + type: SettingInputType.CUSTOM; + key: SettKey; + component: + | string + | React.FC<{ + value: string | boolean | number; + onChange: (value: string) => void; + }>; +} + +interface SettingSection { + title: React.ReactElement; + fields: (SettingFieldInput | SettingFieldCustom)[]; +} + +const ICON_CLASSNAME = 'w-4 h-4 mr-1 inline'; + +const SETTING_SECTIONS: SettingSection[] = [ + { + title: ( + <> + + General + + ), + fields: [ + { + type: SettingInputType.SHORT_INPUT, + label: 'API Key', + key: 'apiKey', + }, + { + type: SettingInputType.LONG_INPUT, + label: 'System Message (will be disabled if left empty)', + key: 'systemMessage', + }, + ...BASIC_KEYS.map( + (key) => + ({ + type: SettingInputType.SHORT_INPUT, + label: key, + key, + }) as SettingFieldInput + ), + ], + }, + { + title: ( + <> + + Samplers + + ), + fields: [ + { + type: SettingInputType.SHORT_INPUT, + label: 'Samplers queue', + key: 'samplers', + }, + ...SAMPLER_KEYS.map( + (key) => + ({ + type: SettingInputType.SHORT_INPUT, + label: key, + key, + }) as SettingFieldInput + ), + ], + }, + { + title: ( + <> + + Penalties + + ), + fields: PENALTY_KEYS.map((key) => ({ + type: SettingInputType.SHORT_INPUT, + label: key, + key, + })), + }, + { + title: ( + <> + + Reasoning + + ), + fields: [ + { + type: SettingInputType.CHECKBOX, + label: 'Expand though process by default for generating message', + key: 'showThoughtInProgress', + }, + { + type: SettingInputType.CHECKBOX, + label: + 'Exclude thought process when sending request to API (Recommended for DeepSeek-R1)', + key: 'excludeThoughtOnReq', + }, + ], + }, + { + title: ( + <> + + Advanced + + ), + fields: [ + { + type: SettingInputType.CUSTOM, + key: 'custom', // dummy key, won't be used + component: () => { + const debugImportDemoConv = async () => { + const res = await fetch('/demo-conversation.json'); + const demoConv = await res.json(); + StorageUtils.remove(demoConv.id); + for (const msg of demoConv.messages) { + StorageUtils.appendMsg(demoConv.id, msg); + } + }; + return ( + + ); + }, + }, + { + type: SettingInputType.CHECKBOX, + label: 'Show tokens per second', + key: 'showTokensPerSecond', + }, + { + type: SettingInputType.LONG_INPUT, + label: ( + <> + Custom JSON config (For more info, refer to{' '} + + server documentation + + ) + + ), + key: 'custom', + }, + ], + }, + { + title: ( + <> + + Experimental + + ), + fields: [ + { + type: SettingInputType.CUSTOM, + key: 'custom', // dummy key, won't be used + component: () => ( + <> +

+ Experimental features are not guaranteed to work correctly. +
+
+ If you encounter any problems, create a{' '} + + Bug (misc.) + {' '} + report on Github. Please also specify webui/experimental on + the report title and include screenshots. +
+
+ Some features may require packages downloaded from CDN, so they + need internet connection. +

+ + ), + }, + { + type: SettingInputType.CHECKBOX, + label: ( + <> + Enable Python interpreter +
+ + This feature uses{' '} + pyodide, + downloaded from CDN. To use this feature, ask the LLM to generate + python code inside a markdown code block. You will see a "Run" + button on the code block, near the "Copy" button. + + + ), + key: 'pyIntepreterEnabled', + }, + ], + }, +]; + export default function SettingDialog({ show, onClose, @@ -40,6 +266,7 @@ export default function SettingDialog({ onClose: () => void; }) { const { config, saveConfig } = useAppContext(); + const [sectionIdx, setSectionIdx] = useState(0); // clone the config object to prevent direct mutation const [localConfig, setLocalConfig] = useState( @@ -92,181 +319,109 @@ export default function SettingDialog({ onClose(); }; - const debugImportDemoConv = async () => { - const res = await fetch('/demo-conversation.json'); - const demoConv = await res.json(); - StorageUtils.remove(demoConv.id); - for (const msg of demoConv.messages) { - StorageUtils.appendMsg(demoConv.id, msg); - } - onClose(); - }; - const onChange = (key: SettKey) => (value: string | boolean) => { // note: we do not perform validation here, because we may get incomplete value as user is still typing it setLocalConfig({ ...localConfig, [key]: value }); }; return ( - -
+ +

Settings

-
-

- Settings below are saved in browser's localStorage -

- - - -