Merge and update

examples/server/public/index.html

@@ -10,6 +10,11 @@
     body {
       font-family: system-ui;
       font-size: 90%;
+      max-width: 600px;
+      min-width: 300px;
+      line-height: 1.2;
+      margin: 0 auto;
+      padding: 0 0.5em;
     }
 
     #container {
@@ -35,12 +40,37 @@
       padding: 0.5em;
     }
 
-    body {
-      max-width: 600px;
-      min-width: 300px;
-      line-height: 1.2;
-      margin: 0 auto;
-      padding: 0 0.5em;
+    #chat {
+      display: flex;
+      flex-direction: column;
+    }
+
+    .bubble {
+      border: 1px solid;
+      border-radius: 1.0em;
+      padding: 0.5em;
+      max-width: 75%;
+      margin-top: 0;
+    }
+
+    .user {
+      background-color: #161616;
+      color: #d6d6d6;
+      border-bottom-right-radius: 0;
+      place-self: flex-end;
+    }
+
+    .asst {
+      background-color: #d6d6d6;
+      color: #161616;
+      border-bottom-left-radius: 0;
+      place-self: flex-start;
+    }
+
+    .typing {
+      color: #888;
+      text-align: left;
+      font-size: 120%;
     }
 
     p {
@@ -102,7 +132,6 @@
       padding: 0.5em;
     }
 
-
     textarea {
       padding: 5px;
       flex-grow: 1;
@@ -138,34 +167,35 @@
   <script type="module">
     import {
       html, h, signal, effect, computed, render, useSignal, useEffect, useRef
-    } from '/index.js';
+    } from '/index.js'
 
-    import { llama } from '/completion.js';
+    import { llama } from '/completion.js'
 
     const session = signal({
-      prompt: "This is a conversation between user and llama, a friendly chatbot. respond in simple markdown.",
-      template: "{{prompt}}\n\n{{history}}\n{{char}}:",
-      historyTemplate: "{{name}}: {{message}}",
+      system: "A chat between a curious user and a pirate.",
+      system_cfg: "A chat between a curious user and an artificial intelligence assistant.",
+      message: "{{system}}\n\n### Instruction:\n{{user}}\n\n### Response:\n{{assistant}}",
+      stop: ["###"],
       transcript: [],
       type: "chat",
-      char: "llama",
-      user: "User",
     })
 
     const params = signal({
+      cfg_scale: 4.0,
+      frequency_penalty: 0.0, // 0.0 = disabled
+      mirostat_eta: 0.1, // learning rate
+      mirostat_tau: 5, // target entropy
+      mirostat: 0, // 0/1/2
       n_predict: 400,
-      temperature: 0.7,
+      penalize_nl: true,
+      presence_penalty: 0.0, // 0.0 = disabled
       repeat_last_n: 256, // 0 = disable penalty, -1 = context size
       repeat_penalty: 1.18, // 1.0 = disabled
+      temperature: 0.7,
+      tfs_z: 1.0, // 1.0 = disabled
       top_k: 40, // <= 0 to use vocab size
       top_p: 0.5, // 1.0 = disabled
-      tfs_z: 1.0, // 1.0 = disabled
       typical_p: 1.0, // 1.0 = disabled
-      presence_penalty: 0.0, // 0.0 = disabled
-      frequency_penalty: 0.0, // 0.0 = disabled
-      mirostat: 0, // 0/1/2
-      mirostat_tau: 5, // target entropy
-      mirostat_eta: 0.1, // learning rate
     })
 
     const llamaStats = signal(null)
@@ -177,88 +207,92 @@
     const transcriptUpdate = (transcript) => {
       session.value = {
         ...session.value,
-        transcript
+        transcript,
       }
     }
 
     // simple template replace
     const template = (str, extraSettings) => {
-      let settings = session.value;
+      let settings = session.value
       if (extraSettings) {
-        settings = { ...settings, ...extraSettings };
+        settings = { ...settings, ...extraSettings }
       }
-      return String(str).replaceAll(/\{\{(.*?)\}\}/g, (_, key) => template(settings[key]));
+      return String(str).replaceAll(/\{\{(.*?)\}\}/g, (_, key) => template(settings[key]))
     }
 
     // send message to server
     const chat = async (msg) => {
       if (controller.value) {
-        console.log('already running...');
-        return;
+        console.log('already running...')
+        return
       }
-      controller.value = new AbortController();
+      controller.value = new AbortController()
 
-      transcriptUpdate([...session.value.transcript, ["{{user}}", msg]])
-
-      const prompt = template(session.value.template, {
-        message: msg,
-        history: session.value.transcript.flatMap(([name, message]) => template(session.value.historyTemplate, {name, message})).join("\n"),
-      });
-
-      let currentMessage = '';
       const history = session.value.transcript
+      const system = history.length == 0 ? session.value.system : ""
+      transcriptUpdate([...history, { system, user: msg, assistant: "" }])
+
+      const prompt = session.value.transcript.map(t =>
+        template(session.value.message, t)).join("").trimEnd()
+
+      const cfg_negative_prompt = params.value.cfg_scale > 1 ? session.value.transcript.map(t =>
+        template(session.value.message, { ...t, system: session.value.system_cfg })
+      ).join("").trimEnd() : ""
+
+      let currentMessage = ''
 
       const llamaParams = {
         ...params.value,
-        stop: ["</s>", template("{{char}}:"), template("{{user}}:")],
+        cfg_negative_prompt,
+        stop: session.stop,
       }
 
       for await (const chunk of llama(prompt, llamaParams, { controller: controller.value })) {
-        const data = chunk.data;
-        currentMessage += data.content;
+        const data = chunk.data
+        currentMessage += data.content
 
         // remove leading whitespace
         currentMessage = currentMessage.replace(/^\s+/, "")
 
-        transcriptUpdate([...history, ["{{char}}", currentMessage]])
+        transcriptUpdate([...history, { system, user: msg, assistant: currentMessage }])
 
         if (data.stop) {
-          console.log("Completion finished: '", currentMessage, "', summary: ", data);
+          console.log("Completion finished: '", currentMessage, "', summary: ", data)
         }
 
         if (data.timings) {
-          llamaStats.value = data.timings;
+          llamaStats.value = data.timings
         }
       }
 
-      controller.value = null;
+      controller.value = null
     }
 
     function MessageInput() {
       const message = useSignal("")
 
       const stop = (e) => {
-        e.preventDefault();
+        e.preventDefault()
         if (controller.value) {
-          controller.value.abort();
-          controller.value = null;
+          controller.value.abort()
+          controller.value = null
         }
       }
 
       const reset = (e) => {
-        stop(e);
-        transcriptUpdate([]);
+        stop(e)
+        transcriptUpdate([])
       }
 
       const submit = (e) => {
-        stop(e);
-        chat(message.value);
-        message.value = "";
+        stop(e)
+        chat(message.value)
+        message.value = ""
       }
 
       const enterSubmits = (event) => {
         if (event.which === 13 && !event.shiftKey) {
-          submit(event);
+          submit(event)
         }
       }
 
@@ -277,7 +311,7 @@
     }
 
     const ChatLog = (props) => {
-      const messages = session.value.transcript;
+      const messages = session.value.transcript
       const container = useRef(null)
 
       useEffect(() => {
@@ -287,102 +321,132 @@
         }
       }, [messages])
 
-      const chatLine = ([user, msg]) => {
-        return html`<p key=${msg}><strong>${template(user)}:</strong> <${Markdownish} text=${template(msg)} /></p>`
-      };
-
       return html`
         <section id="chat" ref=${container}>
-          ${messages.flatMap(chatLine)}
-        </section>`;
+          ${messages.map(({system, user, assistant}) => html`
+            ${system !== "" && html`<p><em><${Markdownish} text=${system} /></em></p>`}
+            <p class="user bubble"><${Markdownish} text=${user} /></p>
+            ${assistant !== "" ?
+              html`<p class="asst bubble"><${Markdownish} text=${assistant} /></p>` :
+              html`<p class="typing">...</p>`}
+          `)}
+        </section>
+      `
+    }
+
+    const updateSession = (el) => session.value = { ...session.value, [el.target.name]: el.target.value }
+    const updateParams = (el) => params.value = { ...params.value, [el.target.name]: el.target.value }
+    const updateParamsFloat = (el) => params.value = { ...params.value, [el.target.name]: parseFloat(el.target.value) }
+    const updateParamsInt = (el) => params.value = { ...params.value, [el.target.name]: Math.floor(parseFloat(el.target.value)) }
+    const updateArray = (el) => {
+        const [name, index] = el.target.name.split(".")
+        const newarr = session.value[name].map((v, i) => i == index ? el.target.value : v).filter(x => x !== "")
+        session.value = { ...session.value, [name]: newarr }
+    }
+    const appendArray = () => session.value = { ...session.value, stop: [...session.value.stop, ""] }
+
+    const ParamSlider = ({param, min, max, step, children}) => {
+      return html`
+        <div>
+          <label for="${param}">${children}</label>
+          <input type="range" id="${param}" min="${min}" max="${max}" step="${step}" name="${param}" value="${params.value[param]}" oninput=${updateParamsFloat} />
+          <span>${params.value[param]}</span>
+        </div>
+      `
+    }
+
+    const FloatField = ({label, max, min, name, step, value}) => {
+      return html`
+        <div>
+          <label for="${name}">${label}</label>
+          <input type="range" id="${name}" min="${min}" max="${max}" step="${step}" name="${name}" value="${value}" oninput=${updateParamsFloat} />
+          <span>${value}</span>
+        </div>
+      `
+    };
+
+    const IntField = ({label, max, min, name, value}) => {
+      return html`
+        <div>
+          <label for="${name}">${label}</label>
+          <input type="range" id="${name}" min="${min}" max="${max}" name="${name}" value="${value}" oninput=${updateParamsInt} />
+          <span>${value}</span>
+        </div>
+      `
     };
 
     const ConfigForm = (props) => {
-      const updateSession = (el) => session.value = { ...session.value, [el.target.name]: el.target.value }
-      const updateParams = (el) => params.value = { ...params.value, [el.target.name]: el.target.value }
-      const updateParamsFloat = (el) => params.value = { ...params.value, [el.target.name]: parseFloat(el.target.value) }
-      const updateParamsInt = (el) => params.value = { ...params.value, [el.target.name]: Math.floor(parseFloat(el.target.value)) }
-
-      const FloatField = ({label, max, min, name, step, value}) => {
-        return html`
-          <div>
-            <label for="${name}">${label}</label>
-            <input type="range" id="${name}" min="${min}" max="${max}" step="${step}" name="${name}" value="${value}" oninput=${updateParamsFloat} />
-            <span>${value}</span>
-          </div>
-        `
-      };
-
-      const IntField = ({label, max, min, name, value}) => {
-        return html`
-          <div>
-            <label for="${name}">${label}</label>
-            <input type="range" id="${name}" min="${min}" max="${max}" name="${name}" value="${value}" oninput=${updateParamsInt} />
-            <span>${value}</span>
-          </div>
-        `
-      };
-
       return html`
         <form>
           <fieldset>
             <div>
-              <label for="prompt">Prompt</label>
-              <textarea type="text" name="prompt" value="${session.value.prompt}" rows=4 oninput=${updateSession}/>
-            </div>
-          </fieldset>
-
-          <fieldset class="two">
-            <div>
-              <label for="user">User name</label>
-              <input type="text" name="user" value="${session.value.user}" oninput=${updateSession} />
+              <label for="system">System prompt</label>
+              <textarea type="text" name="system" value="${session.value.system}" rows=4 oninput=${updateSession}/>
             </div>
 
             <div>
-              <label for="bot">Bot name</label>
-              <input type="text" name="char" value="${session.value.char}" oninput=${updateSession} />
-            </div>
-          </fieldset>
-
-          <fieldset>
-            <div>
-              <label for="template">Prompt template</label>
-              <textarea id="template" name="template" value="${session.value.template}" rows=4 oninput=${updateSession}/>
+              <label for="message">Message template</label>
+              <textarea type="text" name="message" value="${session.value.message}" rows=7 oninput=${updateSession}/>
             </div>
 
             <div>
-              <label for="template">Chat history template</label>
-              <textarea id="template" name="historyTemplate" value="${session.value.historyTemplate}" rows=1 oninput=${updateSession}/>
+              <label for="stop">Stop strings</label>
+              ${session.value.stop.map((stop, i) => html`
+                <p><input type="text" name="stop.${i}" value="${stop}" oninput=${updateArray}/></p>
+              `)}
+              <input type="button" value="+" onclick=${appendArray} />
             </div>
-          </fieldset>
 
-          <fieldset class="two">
-            ${IntField({label: "Predictions", max: 2048, min: -1, name: "n_predict", value: params.value.n_predict})}
-            ${FloatField({label: "Temperature", max: 1.5, min: 0.0, name: "temperature", step: 0.01, value: params.value.temperature})}
-            ${FloatField({label: "Penalize repeat sequence", max: 2.0, min: 0.0, name: "repeat_penalty", step: 0.01, value: params.value.repeat_penalty})}
-            ${IntField({label: "Consider N tokens for penalize", max: 2048, min: 0, name: "repeat_last_n", value: params.value.repeat_last_n})}
-            ${IntField({label: "Top-K sampling", max: 100, min: -1, name: "top_k", value: params.value.top_k})}
-            ${FloatField({label: "Top-P sampling", max: 1.0, min: 0.0, name: "top_p", step: 0.01, value: params.value.top_p})}
-          </fieldset>
-          <details>
-            <summary>More options</summary>
-            <fieldset class="two">
-              ${FloatField({label: "TFS-Z", max: 1.0, min: 0.0, name: "tfs_z", step: 0.01, value: params.value.tfs_z})}
-              ${FloatField({label: "Typical P", max: 1.0, min: 0.0, name: "typical_p", step: 0.01, value: params.value.typical_p})}
-              ${FloatField({label: "Presence penalty", max: 1.0, min: 0.0, name: "presence_penalty", step: 0.01, value: params.value.presence_penalty})}
-              ${FloatField({label: "Frequency penalty", max: 1.0, min: 0.0, name: "frequency_penalty", step: 0.01, value: params.value.frequency_penalty})}
-            </fieldset>
-            <hr />
-            <fieldset class="three">
+            <${ParamSlider} min=1 max=10 step=0.1 param=cfg_scale>CFG scale<//>
+            ${params.value.cfg_scale > 1 && html`
               <div>
-                <label><input type="radio" name="mirostat" value="0" checked=${params.value.mirostat == 0} oninput=${updateParamsInt} /> no Mirostat</label>
-                <label><input type="radio" name="mirostat" value="1" checked=${params.value.mirostat == 1} oninput=${updateParamsInt} /> Mirostat v1</label>
-                <label><input type="radio" name="mirostat" value="2" checked=${params.value.mirostat == 2} oninput=${updateParamsInt} /> Mirostat v2</label>
+                <label for="system_cfg">CFG System prompt</label>
+                <textarea type="text" name="system_cfg" value="${session.value.system_cfg}" rows=4 oninput=${updateSession}/>
               </div>
-              ${FloatField({label: "Mirostat tau", max: 10.0, min: 0.0, name: "mirostat_tau", step: 0.01, value: params.value.mirostat_tau})}
-              ${FloatField({label: "Mirostat eta", max: 1.0, min: 0.0, name: "mirostat_eta", step: 0.01, value: params.value.mirostat_eta})}
+            `}
+          </fieldset>
+
+          <fieldset class="two">
+            <${ParamSlider} min=1 max=1000 step=1 param=n_predict>Predict N tokens<//>
+            <${ParamSlider} min=0 max=1000 step=1 param=repeat_last_n>Penalize last N tokens<//>
+            ${params.value.repeat_last_n > 0 && html`
+              <${ParamSlider} min=0 max=4 step=0.01 param=repeat_penalty>Penalize repeat sequence<//>
+              <${ParamSlider} min=0 max=4 step=0.01 param=frequency_penalty>Penalize frequent tokens<//>
+              <${ParamSlider} min=0 max=4 step=0.01 param=presence_penalty>Penalize tokens not present in prompt<//>
+            `}
+            <${ParamSlider} min=0 max=2 step=0.01 param=temperature>Temperature<//>
+          </fieldset>
+
+          ${params.value.temperature > 0 && html`
+            <fieldset class="three">
+              <label>
+                <input type=radio name=mirostat checked=${params.value.mirostat == 0} value=0 oninput=${updateParamsFloat} />
+                Temperature
+              </label>
+
+              <label><input type=radio name=mirostat checked=${params.value.mirostat == 1} value=1 oninput=${updateParamsFloat} />
+                Mirostat v1
+              </label>
+
+              <label>
+                <input type=radio name=mirostat checked=${params.value.mirostat == 2} value=2 oninput=${updateParamsFloat} />
+                Mirostat v2
+              </label>
             </fieldset>
-          </details>
+
+            <fieldset class="two">
+              ${params.value.mirostat == 0 && html`
+                <${ParamSlider} min=1 max=1000 step=1 param=top_k>Top K<//>
+                <${ParamSlider} min=0 max=1 step=0.01 param=tfs_z>Tail free sampling<//>
+                <${ParamSlider} min=0 max=1 step=0.01 param=typical_p>Typical P<//>
+                <${ParamSlider} min=0 max=1 step=0.01 param=top_p>Top P<//>
+              `}
+              ${params.value.mirostat > 0 && html`
+                <${ParamSlider} min=0 max=1 step=0.01 param=mirostat_eta>Mirostat eta, learning rate<//>
+                <${ParamSlider} min=0 max=1000 step=1 param=mirostat_tau>Mirostat tau, target entropy<//>
+              `}
+              </fieldset>
+            `}
         </form>
       `
     }
@@ -399,9 +463,9 @@
         .replace(/_(.*?)_/g, '<em>$1</em>')
         .replace(/```.*?\n([\s\S]*?)```/g, '<pre><code>$1</code></pre>')
         .replace(/`(.*?)`/g, '<code>$1</code>')
-        .replace(/\n/gim, '<br />');
-      return html`<span dangerouslySetInnerHTML=${{ __html: md }} />`;
-    };
+        .replace(/\n/gim, '<br />')
+      return html`<span dangerouslySetInnerHTML=${{ __html: md }} />`
+    }
 
     const ModelGenerationInfo = (params) => {
       if (!llamaStats.value) {
@@ -435,10 +499,10 @@
             <p>Powered by <a href="https://github.com/ggerganov/llama.cpp">llama.cpp</a> and <a href="https://ggml.ai">ggml.ai</a>.</p>
           </footer>
         </div>
-      `;
+      `
     }
 
-    render(h(App), document.body);
+    render(h(App), document.body)
   </script>
 </head>