added a kobold API compatible implementation of stopping sequences

expose.h

@@ -1,5 +1,6 @@
 #pragma once
 
+const int stop_token_max = 10;
 struct load_model_inputs
 {
     const int threads;
@@ -24,6 +25,7 @@ struct generation_inputs
     const float top_p;
     const float rep_pen;
     const int rep_pen_range;
+    const char * stop_sequence[stop_token_max];
 };
 struct generation_outputs
 {

gpttype_adapter.cpp

@@ -36,8 +36,8 @@ static std::vector<gpt_vocab::id> last_n_tokens;
 static std::vector<gpt_vocab::id> current_context_tokens;
 static size_t mem_per_token = 0;
 static std::vector<float> logits;
-
 static std::vector<int> smartcontext;
+static std::vector<std::string> stop_sequence;
 
 inline bool IsNanCheck(float f)
 {
@@ -154,6 +154,14 @@ ModelLoadResult gpttype_load_model(const load_model_inputs inputs, FileFormat in
 
 generation_outputs gpttype_generate(const generation_inputs inputs, generation_outputs &output)
 {
+    stop_sequence.clear();
+    for(int x=0;x<stop_token_max;++x)
+    {
+        if(inputs.stop_sequence[x]!="")
+        {
+            stop_sequence.push_back(inputs.stop_sequence[x]);
+        }
+    }
     params.prompt = inputs.prompt;
     params.seed = inputs.seed;
     params.n_predict = inputs.max_length;
@@ -333,9 +341,18 @@ generation_outputs gpttype_generate(const generation_inputs inputs, generation_o
 
             // decrement remaining sampling budget
             --remaining_tokens;
-            
+
-            for (auto id : embd) {
+            for (auto id : embd)
+            {
                 concat_output += vocab.id_to_token[id].c_str();
+                for (const auto &matched : stop_sequence)
+                {
+                    if (concat_output.find(matched) != std::string::npos)
+                    {
+                        remaining_tokens = 0;
+                        break;
+                    }
+                }
             }
         }
         else

koboldcpp.py

@@ -8,6 +8,8 @@ import os
 import argparse
 import json, http.server, threading, socket, sys, time
 
+stop_token_max = 10
+
 class load_model_inputs(ctypes.Structure):
     _fields_ = [("threads", ctypes.c_int),
                 ("max_context_length", ctypes.c_int),
@@ -29,7 +31,8 @@ class generation_inputs(ctypes.Structure):
                 ("top_k", ctypes.c_int),
                 ("top_p", ctypes.c_float),
                 ("rep_pen", ctypes.c_float),
-                ("rep_pen_range", ctypes.c_int)]
+                ("rep_pen_range", ctypes.c_int),
+                ("stop_sequence", ctypes.c_char_p * stop_token_max)]
 
 class generation_outputs(ctypes.Structure):
     _fields_ = [("status", ctypes.c_int),
@@ -85,7 +88,7 @@ def load_model(model_filename,batch_size=8,max_context_length=512,n_parts_overwr
     ret = handle.load_model(inputs)
     return ret
 
-def generate(prompt,max_length=20, max_context_length=512,temperature=0.8,top_k=100,top_p=0.85,rep_pen=1.1,rep_pen_range=128,seed=-1):
+def generate(prompt,max_length=20, max_context_length=512,temperature=0.8,top_k=100,top_p=0.85,rep_pen=1.1,rep_pen_range=128,seed=-1,stop_sequence=[]):
     inputs = generation_inputs()
     outputs = ctypes.create_unicode_buffer(ctypes.sizeof(generation_outputs))
     inputs.prompt = prompt.encode("UTF-8")
@@ -97,6 +100,11 @@ def generate(prompt,max_length=20, max_context_length=512,temperature=0.8,top_k=
     inputs.rep_pen = rep_pen
     inputs.rep_pen_range = rep_pen_range
     inputs.seed = seed
+    for n in range(0,stop_token_max):
+        if n >= len(stop_sequence):
+            inputs.stop_sequence[n] = "".encode("UTF-8")
+        else:
+            inputs.stop_sequence[n] = stop_sequence[n].encode("UTF-8")
     ret = handle.generate(inputs,outputs)
     if(ret.status==1):
         return ret.text.decode("UTF-8","ignore")
@@ -170,6 +178,12 @@ class ServerRequestHandler(http.server.SimpleHTTPRequestHandler):
             self.end_headers()
             self.wfile.write(json.dumps({"values": []}).encode())
             return
+        
+        if self.path.endswith(('/api/v1/info/version', '/api/latest/info/version')):
+            self.send_response(200)
+            self.end_headers()           
+            self.wfile.write(json.dumps({"result":"1.2.2"}).encode())
+            return
 
         self.send_response(404)
         self.end_headers()
@@ -229,7 +243,8 @@ class ServerRequestHandler(http.server.SimpleHTTPRequestHandler):
                     top_p=genparams.get('top_p', 0.85),
                     rep_pen=genparams.get('rep_pen', 1.1),
                     rep_pen_range=genparams.get('rep_pen_range', 128),
-                    seed=-1
+                    seed=-1,
+                    stop_sequence=genparams.get('stop_sequence', [])
                     )
                 print("\nOutput: " + recvtxt)
                 res = {"results": [{"text": recvtxt}]}                            
@@ -242,7 +257,8 @@ class ServerRequestHandler(http.server.SimpleHTTPRequestHandler):
                     top_p=genparams.get('top_p', 0.85),
                     rep_pen=genparams.get('rep_pen', 1.1),
                     rep_pen_range=genparams.get('rep_pen_range', 128),
-                    seed=-1
+                    seed=-1,
+                    stop_sequence=genparams.get('stop_sequence', [])
                     )
                 print("\nOutput: " + recvtxt)
                 res = {"data": {"seqs":[recvtxt]}}

llama_adapter.cpp

@@ -34,6 +34,7 @@ static llama_context *ctx;
 static std::vector<llama_token> last_n_tokens;
 static std::vector<llama_token> current_context_tokens;
 static std::vector<llama_token> smartcontext;
+static std::vector<std::string> stop_sequence;
 
 bool llama_load_model(const load_model_inputs inputs, FileFormat in_file_format)
 {
@@ -81,6 +82,15 @@ bool llama_load_model(const load_model_inputs inputs, FileFormat in_file_format)
 
 generation_outputs llama_generate(const generation_inputs inputs, generation_outputs &output)
 {
+    stop_sequence.clear();
+    for(int x=0;x<stop_token_max;++x)
+    {
+        std::string stopper = inputs.stop_sequence[x];
+        if(stopper!="")
+        {
+            stop_sequence.push_back(stopper);
+        }
+    }
     params.prompt = inputs.prompt;
     params.seed = inputs.seed;
     params.n_predict = inputs.max_length;
@@ -231,6 +241,14 @@ generation_outputs llama_generate(const generation_inputs inputs, generation_out
             --remaining_tokens;
             //printf("\nid:%d word:%s\n",id,llama_token_to_str(ctx, id));
             concat_output += llama_token_to_str(ctx, id);
+            for (const auto &matched : stop_sequence)
+            {
+                if (concat_output.find(matched) != std::string::npos)
+                {
+                    remaining_tokens = 0;
+                    break;
+                }
+            }
         }
         else
         {

model_adapter.cpp

@@ -27,7 +27,21 @@ double timer_check()
     return time_taken;
 }
 
-
+void print_vec(std::vector<std::string> &embd)
+{
+    std::cout << "[";
+    bool first = true;
+    for (auto i : embd)
+    {
+        if (!first)
+        {
+            std::cout << ',';
+        }
+        first = false;        
+        std::cout << i;        
+    }
+    std::cout << "]\n";
+}
 void print_tok_vec(std::vector<int> &embd)
 {
     std::cout << "[";

model_adapter.h

@@ -45,6 +45,7 @@ void timer_start();
 double timer_check();
 void print_tok_vec(std::vector<int> &embd);
 void print_tok_vec(std::vector<float> &embd);
+void print_vec(std::vector<std::string> &embd);
 std::vector<int> LongestCommonSubseq(const std::vector<int> x, const std::vector<int> y);
 bool ArrStartWith(const std::vector<int> targetArray, const std::vector<int> searchSeq);
 int ArrFindIndexOf(const std::vector<int> targetArray, const std::vector<int> searchSeq);