added a kobold API compatible implementation of stopping sequences
This commit is contained in:
parent
8bf2e50a11
commit
525184930d
7 changed files with 79 additions and 11 deletions
2
expose.h
2
expose.h
|
@ -1,5 +1,6 @@
|
|||
#pragma once
|
||||
|
||||
const int stop_token_max = 10;
|
||||
struct load_model_inputs
|
||||
{
|
||||
const int threads;
|
||||
|
@ -24,6 +25,7 @@ struct generation_inputs
|
|||
const float top_p;
|
||||
const float rep_pen;
|
||||
const int rep_pen_range;
|
||||
const char * stop_sequence[stop_token_max];
|
||||
};
|
||||
struct generation_outputs
|
||||
{
|
||||
|
|
|
@ -36,8 +36,8 @@ static std::vector<gpt_vocab::id> last_n_tokens;
|
|||
static std::vector<gpt_vocab::id> current_context_tokens;
|
||||
static size_t mem_per_token = 0;
|
||||
static std::vector<float> logits;
|
||||
|
||||
static std::vector<int> smartcontext;
|
||||
static std::vector<std::string> stop_sequence;
|
||||
|
||||
inline bool IsNanCheck(float f)
|
||||
{
|
||||
|
@ -154,6 +154,14 @@ ModelLoadResult gpttype_load_model(const load_model_inputs inputs, FileFormat in
|
|||
|
||||
generation_outputs gpttype_generate(const generation_inputs inputs, generation_outputs &output)
|
||||
{
|
||||
stop_sequence.clear();
|
||||
for(int x=0;x<stop_token_max;++x)
|
||||
{
|
||||
if(inputs.stop_sequence[x]!="")
|
||||
{
|
||||
stop_sequence.push_back(inputs.stop_sequence[x]);
|
||||
}
|
||||
}
|
||||
params.prompt = inputs.prompt;
|
||||
params.seed = inputs.seed;
|
||||
params.n_predict = inputs.max_length;
|
||||
|
@ -333,9 +341,18 @@ generation_outputs gpttype_generate(const generation_inputs inputs, generation_o
|
|||
|
||||
// decrement remaining sampling budget
|
||||
--remaining_tokens;
|
||||
|
||||
for (auto id : embd) {
|
||||
|
||||
for (auto id : embd)
|
||||
{
|
||||
concat_output += vocab.id_to_token[id].c_str();
|
||||
for (const auto &matched : stop_sequence)
|
||||
{
|
||||
if (concat_output.find(matched) != std::string::npos)
|
||||
{
|
||||
remaining_tokens = 0;
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
else
|
||||
|
|
File diff suppressed because one or more lines are too long
24
koboldcpp.py
24
koboldcpp.py
|
@ -8,6 +8,8 @@ import os
|
|||
import argparse
|
||||
import json, http.server, threading, socket, sys, time
|
||||
|
||||
stop_token_max = 10
|
||||
|
||||
class load_model_inputs(ctypes.Structure):
|
||||
_fields_ = [("threads", ctypes.c_int),
|
||||
("max_context_length", ctypes.c_int),
|
||||
|
@ -29,7 +31,8 @@ class generation_inputs(ctypes.Structure):
|
|||
("top_k", ctypes.c_int),
|
||||
("top_p", ctypes.c_float),
|
||||
("rep_pen", ctypes.c_float),
|
||||
("rep_pen_range", ctypes.c_int)]
|
||||
("rep_pen_range", ctypes.c_int),
|
||||
("stop_sequence", ctypes.c_char_p * stop_token_max)]
|
||||
|
||||
class generation_outputs(ctypes.Structure):
|
||||
_fields_ = [("status", ctypes.c_int),
|
||||
|
@ -85,7 +88,7 @@ def load_model(model_filename,batch_size=8,max_context_length=512,n_parts_overwr
|
|||
ret = handle.load_model(inputs)
|
||||
return ret
|
||||
|
||||
def generate(prompt,max_length=20, max_context_length=512,temperature=0.8,top_k=100,top_p=0.85,rep_pen=1.1,rep_pen_range=128,seed=-1):
|
||||
def generate(prompt,max_length=20, max_context_length=512,temperature=0.8,top_k=100,top_p=0.85,rep_pen=1.1,rep_pen_range=128,seed=-1,stop_sequence=[]):
|
||||
inputs = generation_inputs()
|
||||
outputs = ctypes.create_unicode_buffer(ctypes.sizeof(generation_outputs))
|
||||
inputs.prompt = prompt.encode("UTF-8")
|
||||
|
@ -97,6 +100,11 @@ def generate(prompt,max_length=20, max_context_length=512,temperature=0.8,top_k=
|
|||
inputs.rep_pen = rep_pen
|
||||
inputs.rep_pen_range = rep_pen_range
|
||||
inputs.seed = seed
|
||||
for n in range(0,stop_token_max):
|
||||
if n >= len(stop_sequence):
|
||||
inputs.stop_sequence[n] = "".encode("UTF-8")
|
||||
else:
|
||||
inputs.stop_sequence[n] = stop_sequence[n].encode("UTF-8")
|
||||
ret = handle.generate(inputs,outputs)
|
||||
if(ret.status==1):
|
||||
return ret.text.decode("UTF-8","ignore")
|
||||
|
@ -170,6 +178,12 @@ class ServerRequestHandler(http.server.SimpleHTTPRequestHandler):
|
|||
self.end_headers()
|
||||
self.wfile.write(json.dumps({"values": []}).encode())
|
||||
return
|
||||
|
||||
if self.path.endswith(('/api/v1/info/version', '/api/latest/info/version')):
|
||||
self.send_response(200)
|
||||
self.end_headers()
|
||||
self.wfile.write(json.dumps({"result":"1.2.2"}).encode())
|
||||
return
|
||||
|
||||
self.send_response(404)
|
||||
self.end_headers()
|
||||
|
@ -229,7 +243,8 @@ class ServerRequestHandler(http.server.SimpleHTTPRequestHandler):
|
|||
top_p=genparams.get('top_p', 0.85),
|
||||
rep_pen=genparams.get('rep_pen', 1.1),
|
||||
rep_pen_range=genparams.get('rep_pen_range', 128),
|
||||
seed=-1
|
||||
seed=-1,
|
||||
stop_sequence=genparams.get('stop_sequence', [])
|
||||
)
|
||||
print("\nOutput: " + recvtxt)
|
||||
res = {"results": [{"text": recvtxt}]}
|
||||
|
@ -242,7 +257,8 @@ class ServerRequestHandler(http.server.SimpleHTTPRequestHandler):
|
|||
top_p=genparams.get('top_p', 0.85),
|
||||
rep_pen=genparams.get('rep_pen', 1.1),
|
||||
rep_pen_range=genparams.get('rep_pen_range', 128),
|
||||
seed=-1
|
||||
seed=-1,
|
||||
stop_sequence=genparams.get('stop_sequence', [])
|
||||
)
|
||||
print("\nOutput: " + recvtxt)
|
||||
res = {"data": {"seqs":[recvtxt]}}
|
||||
|
|
|
@ -34,6 +34,7 @@ static llama_context *ctx;
|
|||
static std::vector<llama_token> last_n_tokens;
|
||||
static std::vector<llama_token> current_context_tokens;
|
||||
static std::vector<llama_token> smartcontext;
|
||||
static std::vector<std::string> stop_sequence;
|
||||
|
||||
bool llama_load_model(const load_model_inputs inputs, FileFormat in_file_format)
|
||||
{
|
||||
|
@ -81,6 +82,15 @@ bool llama_load_model(const load_model_inputs inputs, FileFormat in_file_format)
|
|||
|
||||
generation_outputs llama_generate(const generation_inputs inputs, generation_outputs &output)
|
||||
{
|
||||
stop_sequence.clear();
|
||||
for(int x=0;x<stop_token_max;++x)
|
||||
{
|
||||
std::string stopper = inputs.stop_sequence[x];
|
||||
if(stopper!="")
|
||||
{
|
||||
stop_sequence.push_back(stopper);
|
||||
}
|
||||
}
|
||||
params.prompt = inputs.prompt;
|
||||
params.seed = inputs.seed;
|
||||
params.n_predict = inputs.max_length;
|
||||
|
@ -231,6 +241,14 @@ generation_outputs llama_generate(const generation_inputs inputs, generation_out
|
|||
--remaining_tokens;
|
||||
//printf("\nid:%d word:%s\n",id,llama_token_to_str(ctx, id));
|
||||
concat_output += llama_token_to_str(ctx, id);
|
||||
for (const auto &matched : stop_sequence)
|
||||
{
|
||||
if (concat_output.find(matched) != std::string::npos)
|
||||
{
|
||||
remaining_tokens = 0;
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
else
|
||||
{
|
||||
|
|
|
@ -27,7 +27,21 @@ double timer_check()
|
|||
return time_taken;
|
||||
}
|
||||
|
||||
|
||||
void print_vec(std::vector<std::string> &embd)
|
||||
{
|
||||
std::cout << "[";
|
||||
bool first = true;
|
||||
for (auto i : embd)
|
||||
{
|
||||
if (!first)
|
||||
{
|
||||
std::cout << ',';
|
||||
}
|
||||
first = false;
|
||||
std::cout << i;
|
||||
}
|
||||
std::cout << "]\n";
|
||||
}
|
||||
void print_tok_vec(std::vector<int> &embd)
|
||||
{
|
||||
std::cout << "[";
|
||||
|
|
|
@ -45,6 +45,7 @@ void timer_start();
|
|||
double timer_check();
|
||||
void print_tok_vec(std::vector<int> &embd);
|
||||
void print_tok_vec(std::vector<float> &embd);
|
||||
void print_vec(std::vector<std::string> &embd);
|
||||
std::vector<int> LongestCommonSubseq(const std::vector<int> x, const std::vector<int> y);
|
||||
bool ArrStartWith(const std::vector<int> targetArray, const std::vector<int> searchSeq);
|
||||
int ArrFindIndexOf(const std::vector<int> targetArray, const std::vector<int> searchSeq);
|
||||
|
|
Loading…
Add table
Add a link
Reference in a new issue