vbatts/llama.cpp
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Merge remote-tracking branch 'temp/concedo' into concedo_experimental

Browse source
This commit is contained in:
Concedo 2023-05-05 23:55:53 +08:00
commit 851f55325a
3 changed files with 20 additions and 2 deletions
Download patch file Download diff file Expand all files Collapse all files

5
expose.h
View file

@ -33,6 +33,9 @@ struct generation_inputs
const float tfs; const float tfs;
const float rep_pen; const float rep_pen;
const int rep_pen_range; const int rep_pen_range;
const int mirostat;
const float mirostat_eta;
const float mirostat_tau;
const char * stop_sequence[stop_token_max]; const char * stop_sequence[stop_token_max];
}; };
struct generation_outputs struct generation_outputs
@ -42,4 +45,4 @@ struct generation_outputs
}; };
extern std::string executable_path; extern std::string executable_path;
extern std::string lora_filename; extern std::string lora_filename;

3
gpttype_adapter.cpp
View file

@ -371,6 +371,9 @@ generation_outputs gpttype_generate(const generation_inputs inputs, generation_o
params.temp = inputs.temperature; params.temp = inputs.temperature;
params.repeat_last_n = inputs.rep_pen_range; params.repeat_last_n = inputs.rep_pen_range;
params.repeat_penalty = inputs.rep_pen; params.repeat_penalty = inputs.rep_pen;
params.mirostat = inputs.mirostat;
params.mirostat_eta = inputs.mirostat_eta;
params.mirostat_tau = inputs.mirostat_tau;
params.n_ctx = inputs.max_context_length; params.n_ctx = inputs.max_context_length;
params.n_batch = n_batch; params.n_batch = n_batch;
params.n_threads = n_threads; params.n_threads = n_threads;

14
koboldcpp.py
View file

@ -39,6 +39,9 @@ class generation_inputs(ctypes.Structure):
("tfs", ctypes.c_float), ("tfs", ctypes.c_float),
("rep_pen", ctypes.c_float), ("rep_pen", ctypes.c_float),
("rep_pen_range", ctypes.c_int), ("rep_pen_range", ctypes.c_int),
("mirostat", ctypes.c_int),
("mirostat_tau", ctypes.c_float),
("mirostat_eta", ctypes.c_float),
("stop_sequence", ctypes.c_char_p * stop_token_max)] ("stop_sequence", ctypes.c_char_p * stop_token_max)]
class generation_outputs(ctypes.Structure): class generation_outputs(ctypes.Structure):
@ -154,7 +157,7 @@ def load_model(model_filename):
ret = handle.load_model(inputs) ret = handle.load_model(inputs)
return ret return ret
def generate(prompt,max_length=20, max_context_length=512,temperature=0.8,top_k=100,top_p=0.85, typical_p=1.0, tfs=1.0 ,rep_pen=1.1,rep_pen_range=128,seed=-1,stop_sequence=[]): def generate(prompt,max_length=20, max_context_length=512,temperature=0.8,top_k=100,top_p=0.85, typical_p=1.0, tfs=1.0 ,rep_pen=1.1,rep_pen_range=128,mirostat=0,mirostat_lr=0.1,mirostat_ent=5.0,seed=-1,stop_sequence=[]):
inputs = generation_inputs() inputs = generation_inputs()
outputs = ctypes.create_unicode_buffer(ctypes.sizeof(generation_outputs)) outputs = ctypes.create_unicode_buffer(ctypes.sizeof(generation_outputs))
inputs.prompt = prompt.encode("UTF-8") inputs.prompt = prompt.encode("UTF-8")
@ -167,6 +170,9 @@ def generate(prompt,max_length=20, max_context_length=512,temperature=0.8,top_k=
inputs.tfs = tfs inputs.tfs = tfs
inputs.rep_pen = rep_pen inputs.rep_pen = rep_pen
inputs.rep_pen_range = rep_pen_range inputs.rep_pen_range = rep_pen_range
inputs.mirostat = mirostat
inputs.mirostat_eta = mirostat_lr
inputs.mirostat_tau = mirostat_ent
inputs.seed = seed inputs.seed = seed
for n in range(0,stop_token_max): for n in range(0,stop_token_max):
if not stop_sequence or n >= len(stop_sequence): if not stop_sequence or n >= len(stop_sequence):
@ -311,6 +317,9 @@ class ServerRequestHandler(http.server.SimpleHTTPRequestHandler):
tfs=genparams.get('tfs', 1.0), tfs=genparams.get('tfs', 1.0),
rep_pen=genparams.get('rep_pen', 1.1), rep_pen=genparams.get('rep_pen', 1.1),
rep_pen_range=genparams.get('rep_pen_range', 128), rep_pen_range=genparams.get('rep_pen_range', 128),
mirostat=genparams.get('mirostat', 0),
mirostat_lr=genparams.get('mirostat_lr', 0.1),
mirostat_ent=genparams.get('mirostat_ent', 5.0),
seed=-1, seed=-1,
stop_sequence=genparams.get('stop_sequence', []) stop_sequence=genparams.get('stop_sequence', [])
) )
@ -327,6 +336,9 @@ class ServerRequestHandler(http.server.SimpleHTTPRequestHandler):
tfs=genparams.get('tfs', 1.0), tfs=genparams.get('tfs', 1.0),
rep_pen=genparams.get('rep_pen', 1.1), rep_pen=genparams.get('rep_pen', 1.1),
rep_pen_range=genparams.get('rep_pen_range', 128), rep_pen_range=genparams.get('rep_pen_range', 128),
mirostat=genparams.get('mirostat', 0),
mirostat_lr=genparams.get('mirostat_lr', 0.1),
mirostat_ent=genparams.get('mirostat_ent', 5.0),
seed=-1, seed=-1,
stop_sequence=genparams.get('stop_sequence', []) stop_sequence=genparams.get('stop_sequence', [])
) )