vbatts/llama.cpp
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

add new sampling algorithm mirostat

Browse source
This commit is contained in:
Hendrik Langer 2023-05-05 13:23:47 +02:00
parent c8f7eeb7fd
commit 8131bc8b56
3 changed files with 20 additions and 2 deletions
Download patch file Download diff file Expand all files Collapse all files

3
expose.h
View file

@ -32,6 +32,9 @@ struct generation_inputs
const float tfs;
const float rep_pen;
const int rep_pen_range;
const int mirostat;
const float mirostat_eta;
const float mirostat_tau;
const char * stop_sequence[stop_token_max];
};
struct generation_outputs

3
gpttype_adapter.cpp
View file

@ -371,6 +371,9 @@ generation_outputs gpttype_generate(const generation_inputs inputs, generation_o
params.temp = inputs.temperature;
params.repeat_last_n = inputs.rep_pen_range;
params.repeat_penalty = inputs.rep_pen;
params.mirostat = inputs.mirostat;
params.mirostat_eta = inputs.mirostat_eta;
params.mirostat_tau = inputs.mirostat_tau;
params.n_ctx = inputs.max_context_length;
params.n_batch = n_batch;
params.n_threads = n_threads;

14
koboldcpp.py
View file

@ -38,6 +38,9 @@ class generation_inputs(ctypes.Structure):
("tfs", ctypes.c_float),
("rep_pen", ctypes.c_float),
("rep_pen_range", ctypes.c_int),
("mirostat", ctypes.c_int),
("mirostat_tau", ctypes.c_float),
("mirostat_eta", ctypes.c_float),
("stop_sequence", ctypes.c_char_p * stop_token_max)]
class generation_outputs(ctypes.Structure):
@ -152,7 +155,7 @@ def load_model(model_filename):
ret = handle.load_model(inputs)
return ret
def generate(prompt,max_length=20, max_context_length=512,temperature=0.8,top_k=100,top_p=0.85, typical_p=1.0, tfs=1.0 ,rep_pen=1.1,rep_pen_range=128,seed=-1,stop_sequence=[]):
def generate(prompt,max_length=20, max_context_length=512,temperature=0.8,top_k=100,top_p=0.85, typical_p=1.0, tfs=1.0 ,rep_pen=1.1,rep_pen_range=128,mirostat=0,mirostat_lr=0.1,mirostat_ent=5.0,seed=-1,stop_sequence=[]):
inputs = generation_inputs()
outputs = ctypes.create_unicode_buffer(ctypes.sizeof(generation_outputs))
inputs.prompt = prompt.encode("UTF-8")
@ -165,6 +168,9 @@ def generate(prompt,max_length=20, max_context_length=512,temperature=0.8,top_k=
inputs.tfs = tfs
inputs.rep_pen = rep_pen
inputs.rep_pen_range = rep_pen_range
inputs.mirostat = mirostat
inputs.mirostat_eta = mirostat_lr
inputs.mirostat_tau = mirostat_ent
inputs.seed = seed
for n in range(0,stop_token_max):
if not stop_sequence or n >= len(stop_sequence):
@ -309,6 +315,9 @@ class ServerRequestHandler(http.server.SimpleHTTPRequestHandler):
tfs=genparams.get('tfs', 1.0),
rep_pen=genparams.get('rep_pen', 1.1),
rep_pen_range=genparams.get('rep_pen_range', 128),
mirostat=genparams.get('mirostat', 0),
mirostat_lr=genparams.get('mirostat_lr', 0.1),
mirostat_ent=genparams.get('mirostat_ent', 5.0),
seed=-1,
stop_sequence=genparams.get('stop_sequence', [])
)
@ -325,6 +334,9 @@ class ServerRequestHandler(http.server.SimpleHTTPRequestHandler):
tfs=genparams.get('tfs', 1.0),
rep_pen=genparams.get('rep_pen', 1.1),
rep_pen_range=genparams.get('rep_pen_range', 128),
mirostat=genparams.get('mirostat', 0),
mirostat_lr=genparams.get('mirostat_lr', 0.1),
mirostat_ent=genparams.get('mirostat_ent', 5.0),
seed=-1,
stop_sequence=genparams.get('stop_sequence', [])
)