diff --git a/expose.h b/expose.h index e8550cb7f..c3707aa19 100644 --- a/expose.h +++ b/expose.h @@ -32,6 +32,9 @@ struct generation_inputs const float tfs; const float rep_pen; const int rep_pen_range; + const int mirostat; + const float mirostat_eta; + const float mirostat_tau; const char * stop_sequence[stop_token_max]; }; struct generation_outputs @@ -41,4 +44,4 @@ struct generation_outputs }; extern std::string executable_path; -extern std::string lora_filename; \ No newline at end of file +extern std::string lora_filename; diff --git a/gpttype_adapter.cpp b/gpttype_adapter.cpp index 97b95d57e..e90144d97 100644 --- a/gpttype_adapter.cpp +++ b/gpttype_adapter.cpp @@ -371,6 +371,9 @@ generation_outputs gpttype_generate(const generation_inputs inputs, generation_o params.temp = inputs.temperature; params.repeat_last_n = inputs.rep_pen_range; params.repeat_penalty = inputs.rep_pen; + params.mirostat = inputs.mirostat; + params.mirostat_eta = inputs.mirostat_eta; + params.mirostat_tau = inputs.mirostat_tau; params.n_ctx = inputs.max_context_length; params.n_batch = n_batch; params.n_threads = n_threads; diff --git a/koboldcpp.py b/koboldcpp.py index 2ac195ef9..14cf226a6 100644 --- a/koboldcpp.py +++ b/koboldcpp.py @@ -38,6 +38,9 @@ class generation_inputs(ctypes.Structure): ("tfs", ctypes.c_float), ("rep_pen", ctypes.c_float), ("rep_pen_range", ctypes.c_int), + ("mirostat", ctypes.c_int), + ("mirostat_tau", ctypes.c_float), + ("mirostat_eta", ctypes.c_float), ("stop_sequence", ctypes.c_char_p * stop_token_max)] class generation_outputs(ctypes.Structure): @@ -152,7 +155,7 @@ def load_model(model_filename): ret = handle.load_model(inputs) return ret -def generate(prompt,max_length=20, max_context_length=512,temperature=0.8,top_k=100,top_p=0.85, typical_p=1.0, tfs=1.0 ,rep_pen=1.1,rep_pen_range=128,seed=-1,stop_sequence=[]): +def generate(prompt,max_length=20, max_context_length=512,temperature=0.8,top_k=100,top_p=0.85, typical_p=1.0, tfs=1.0 ,rep_pen=1.1,rep_pen_range=128,mirostat=0,mirostat_lr=0.1,mirostat_ent=5.0,seed=-1,stop_sequence=[]): inputs = generation_inputs() outputs = ctypes.create_unicode_buffer(ctypes.sizeof(generation_outputs)) inputs.prompt = prompt.encode("UTF-8") @@ -165,6 +168,9 @@ def generate(prompt,max_length=20, max_context_length=512,temperature=0.8,top_k= inputs.tfs = tfs inputs.rep_pen = rep_pen inputs.rep_pen_range = rep_pen_range + inputs.mirostat = mirostat + inputs.mirostat_eta = mirostat_lr + inputs.mirostat_tau = mirostat_ent inputs.seed = seed for n in range(0,stop_token_max): if not stop_sequence or n >= len(stop_sequence): @@ -309,6 +315,9 @@ class ServerRequestHandler(http.server.SimpleHTTPRequestHandler): tfs=genparams.get('tfs', 1.0), rep_pen=genparams.get('rep_pen', 1.1), rep_pen_range=genparams.get('rep_pen_range', 128), + mirostat=genparams.get('mirostat', 0), + mirostat_lr=genparams.get('mirostat_lr', 0.1), + mirostat_ent=genparams.get('mirostat_ent', 5.0), seed=-1, stop_sequence=genparams.get('stop_sequence', []) ) @@ -325,6 +334,9 @@ class ServerRequestHandler(http.server.SimpleHTTPRequestHandler): tfs=genparams.get('tfs', 1.0), rep_pen=genparams.get('rep_pen', 1.1), rep_pen_range=genparams.get('rep_pen_range', 128), + mirostat=genparams.get('mirostat', 0), + mirostat_lr=genparams.get('mirostat_lr', 0.1), + mirostat_ent=genparams.get('mirostat_ent', 5.0), seed=-1, stop_sequence=genparams.get('stop_sequence', []) )