vbatts/llama.cpp
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

kobold integration of min_p sampler (+1 squashed commits)

Browse source 
Squashed commits:

[8ad2e349] kobold integration for min_p sampler
This commit is contained in:
Concedo 2023-11-01 19:07:26 +08:00
parent bcb397953f
commit ae2cd56de8
5 changed files with 38 additions and 7 deletions
Download patch file Download diff file Expand all files Collapse all files

1
common/common.h
View file

@ -58,6 +58,7 @@ struct gpt_params {
// sampling parameters
int32_t top_k = 40; // <= 0 to use vocab size
float top_p = 0.95f; // 1.0 = disabled
float min_p = 0.0f; // 0.0 = disabled
float tfs_z = 1.00f; // 1.0 = disabled
float typical_p = 1.00f; // 1.0 = disabled
float temp = 0.80f; // 1.0 = disabled

1
expose.h
View file

@ -60,6 +60,7 @@ struct generation_inputs
const int top_k;
const float top_a = 0.0f;
const float top_p;
const float min_p = 0.0f;
const float typical_p;
const float tfs;
const float rep_pen;

7
gpttype_adapter.cpp
View file

@ -470,7 +470,7 @@ void sample_grammar(FileFormat file_format, int32_t n_vocab, llama_token_data_ar
}
int SampleLogits(const float * logits, int n_ctx, int n_vocab, int rep_pen_range, float rep_pen, float top_k, float top_a, float top_p, float typical_p, float tfs, float temp, std::mt19937 & rng,
int SampleLogits(const float * logits, int n_ctx, int n_vocab, int rep_pen_range, float rep_pen, float top_k, float top_a, float top_p, float min_p, float typical_p, float tfs, float temp, std::mt19937 & rng,
int mirostat, float mirostat_tau, float mirostat_eta, const std::vector<samplers> & sampler_order, llama_grammar * grammar)
{
int id = 0;
@ -515,6 +515,7 @@ int mirostat, float mirostat_tau, float mirostat_eta, const std::vector<samplers
break;
case KCPP_SAMPLER_TOP_P:
llama_sample_top_p(nullptr, &candidates_p, top_p,1);
llama_sample_min_p(nullptr, &candidates_p, min_p,1);
break;
case KCPP_SAMPLER_TFS:
llama_sample_tail_free(nullptr, &candidates_p, tfs,1);
@ -1392,6 +1393,7 @@ generation_outputs gpttype_generate(const generation_inputs inputs, generation_o
params.n_predict = inputs.max_length;
params.top_k = inputs.top_k;
params.top_p = inputs.top_p;
params.min_p = inputs.min_p;
params.typical_p = inputs.typical_p;
params.tfs_z = inputs.tfs;
params.temp = inputs.temperature;
@ -1740,6 +1742,7 @@ generation_outputs gpttype_generate(const generation_inputs inputs, generation_o
// out of user input, sample next token
const float top_k = params.top_k;
const float top_p = params.top_p;
const float min_p = params.min_p;
const float temp = params.temp;
const float top_a = inputs.top_a;
const float repeat_penalty = params.repeat_penalty;
@ -1799,7 +1802,7 @@ generation_outputs gpttype_generate(const generation_inputs inputs, generation_o
}
id = SampleLogits(logitsPtr, nctx, n_vocab, last_n_size, repeat_penalty,
top_k, top_a, top_p, typical_p, tfs_z, temp, rng,
top_k, top_a, top_p, min_p, typical_p, tfs_z, temp, rng,
params.mirostat, params.mirostat_tau, params.mirostat_eta, sampler_order, grammar);
if (grammar != nullptr) {

25
klite.embd
View file

@ -3315,6 +3315,7 @@ Current version: 89
rep_pen_slope: 0.7,
temperature: 0.7,
top_p: 0.92,
min_p: 0.00,
top_k: 100,
top_a: 0,
typ_s: 1,
@ -3336,6 +3337,7 @@ Current version: 89
genamt: defaultsettings.max_length,
top_k: defaultsettings.top_k,
top_p: defaultsettings.top_p,
min_p: defaultsettings.min_p,
top_a: defaultsettings.top_a,
typical: defaultsettings.typ_s,
tfs: defaultsettings.tfs_s,
@ -3351,6 +3353,7 @@ Current version: 89
genamt: defaultsettings.max_length,
top_k: defaultsettings.top_k,
top_p: defaultsettings.top_p,
min_p: defaultsettings.min_p,
top_a: defaultsettings.top_a,
typical: defaultsettings.typ_s,
tfs: defaultsettings.tfs_s,
@ -3359,7 +3362,7 @@ Current version: 89
rep_pen_slope: defaultsettings.rep_pen_slope,
sampler_order: [0, 1, 2, 3, 4, 5, 6]
},
{"preset":"Godlike","description":"Makes AI give a descriptive and sensual output.","temp":0.7,"genamt":80,"top_k":0,"top_p":0.5,"top_a":0.75,"typical":0.19,"tfs":0.97,"rep_pen":1.1,"rep_pen_range":1024,"rep_pen_slope":0.7,"sampler_order":[6,5,4,3,2,1,0]},{"preset":"Mayday","description":"Wacky plot, creativity from AI, crazy stories you want AI to weird out.","temp":1.05,"genamt":80,"top_k":0,"top_p":0.95,"top_a":0,"typical":1,"tfs":1,"rep_pen":1.1,"rep_pen_range":1024,"rep_pen_slope":0.7,"sampler_order":[6,0,1,2,3,4,5]},{"preset":"Good Winds","description":"Let AI direct the plot, but still stay logical.","temp":0.7,"genamt":80,"top_k":0,"top_p":1,"top_a":0,"typical":1,"tfs":0.9,"rep_pen":1.1,"rep_pen_range":1024,"rep_pen_slope":0.7,"sampler_order":[6,0,1,2,3,4,5]},{"preset":"Liminal Drift","description":"Drives coherent dialogue, responses, and behavior, sometimes surreal situations arise based on information already present in the story.","temp":0.66,"genamt":80,"top_k":0,"top_p":1,"top_a":0.96,"typical":0.6,"tfs":1,"rep_pen":1.1,"rep_pen_range":1024,"rep_pen_slope":0.7,"sampler_order":[6,4,5,1,0,2,3]},{"preset":"TavernAI","description":"Preset used in TavernAI.","temp":0.79,"genamt":80,"top_k":0,"top_p":0.9,"top_a":0,"typical":1,"tfs":0.95,"rep_pen":1.19,"rep_pen_range":1024,"rep_pen_slope":0.9,"sampler_order":[6,0,1,2,3,4,5]},{"preset":"Storywriter 6B","description":"Optimized settings for relevant output.","genamt":80,"rep_pen":1.1,"rep_pen_range":2048,"rep_pen_slope":0.2,"sampler_order":[6,5,0,2,3,1,4],"temp":0.72,"tfs":1,"top_a":0,"top_k":0,"top_p":0.73,"typical":1},{"preset":"Coherent Creativity 6B","description":"A good balance between coherence, creativity, and quality of prose.","genamt":80,"rep_pen":1.2,"rep_pen_range":2048,"rep_pen_slope":0,"sampler_order":[6,5,0,2,3,1,4],"temp":0.51,"tfs":0.99,"top_a":0,"top_k":0,"top_p":1,"typical":1},{"preset":"Luna Moth 6B","description":"A great degree of creativity without losing coherency.","temp":1.5,"genamt":80,"top_k":85,"top_p":0.24,"top_a":0,"typical":1,"tfs":1,"rep_pen":1.1,"rep_pen_range":2048,"rep_pen_slope":0,"sampler_order":[6,5,0,2,3,1,4]},{"preset":"Best Guess 6B","description":"A subtle change with alternative context settings.","temp":0.8,"genamt":80,"top_k":100,"top_p":0.9,"top_a":0,"typical":1,"tfs":1,"rep_pen":1.15,"rep_pen_range":2048,"rep_pen_slope":3.4,"sampler_order":[6,5,0,2,3,1,4]},{"preset":"Pleasing Results 6B","description":"Expectable output with alternative context settings.","temp":0.44,"genamt":80,"top_k":0,"top_p":1,"top_a":0,"typical":1,"tfs":0.9,"rep_pen":1.15,"rep_pen_range":2048,"rep_pen_slope":6.8,"sampler_order":[6,5,0,2,3,1,4]},{"preset":"Genesis 13B","description":"Stable and logical, but with scattered creativity.","temp":0.63,"genamt":80,"top_k":0,"top_p":0.98,"top_a":0,"typical":1,"tfs":0.98,"rep_pen":1.05,"rep_pen_range":2048,"rep_pen_slope":0.1,"sampler_order":[6,2,0,3,5,1,4]},{"preset":"Basic Coherence 13B","description":"Keep things on track.","temp":0.59,"genamt":80,"top_k":0,"top_p":1,"top_a":0,"typical":1,"tfs":0.87,"rep_pen":1.1,"rep_pen_range":2048,"rep_pen_slope":0.3,"sampler_order":[6,5,0,2,3,1,4]},{"preset":"Ouroboros 13B","description":"Versatile, conforms well to poems, lists, chat, etc.","temp":1.07,"genamt":80,"top_k":100,"top_p":1,"top_a":0,"typical":1,"tfs":0.93,"rep_pen":1.05,"rep_pen_range":404,"rep_pen_slope":0.8,"sampler_order":[6,0,5,3,2,1,4]},{"preset":"Ace of Spades 13B","description":"Expressive, while still staying focused.","temp":1.15,"genamt":80,"top_k":0,"top_p":0.95,"top_a":0,"typical":1,"tfs":0.8,"rep_pen":1.05,"rep_pen_range":2048,"rep_pen_slope":7,"sampler_order":[6,3,2,0,5,1,4]},{"preset":"Low Rider 13B","description":"Reliable, aimed at story development.","temp":0.94,"genamt":80,"top_k":12,"top_p":1,"top_a":0,"typical":1,"tfs":0.94,"rep_pen":1.05,"rep_pen_range":2048,"rep_pen_slope":0.2,"sampler_order":[6,5,0,2,3,1,4]},{"preset":"Pro Writer 13B","description":"Optimal setting for readability, based on AI-powered mass statistical analysis of Euterpe output.","temp":1.35,"genamt":80,"top_k":0,"top_p":1,"top_a":0,"typical":1,"tfs":0.69,"rep_pen":1.15,"rep_pen_range":2048,"rep_pen_slope":0.1,"sampler_order":[6,3,2,5,0,1,4]},{"preset":"Default 20B","description":"Good starting settings for NeoX 20B.","temp":0.6,"genamt":80,"top_k":0,"top_p":0.9,"top_a":0,"typical":1,"tfs":1,"rep_pen":1.04,"rep_pen_range":1024,"rep_pen_slope":0.7,"sampler_order":[6,0,1,2,3,4,5]}
{"preset":"Godlike","description":"Makes AI give a descriptive and sensual output.","temp":0.7,"genamt":80,"top_k":0,"top_p":0.5,"min_p":0.0,"top_a":0.75,"typical":0.19,"tfs":0.97,"rep_pen":1.1,"rep_pen_range":1024,"rep_pen_slope":0.7,"sampler_order":[6,5,4,3,2,1,0]},{"preset":"Mayday","description":"Wacky plot, creativity from AI, crazy stories you want AI to weird out.","temp":1.05,"genamt":80,"top_k":0,"top_p":0.95,"min_p":0.0,"top_a":0,"typical":1,"tfs":1,"rep_pen":1.1,"rep_pen_range":1024,"rep_pen_slope":0.7,"sampler_order":[6,0,1,2,3,4,5]},{"preset":"Good Winds","description":"Let AI direct the plot, but still stay logical.","temp":0.7,"genamt":80,"top_k":0,"top_p":1,"min_p":0.0,"top_a":0,"typical":1,"tfs":0.9,"rep_pen":1.1,"rep_pen_range":1024,"rep_pen_slope":0.7,"sampler_order":[6,0,1,2,3,4,5]},{"preset":"Liminal Drift","description":"Drives coherent dialogue, responses, and behavior, sometimes surreal situations arise based on information already present in the story.","temp":0.66,"genamt":80,"top_k":0,"top_p":1,"min_p":0.0,"top_a":0.96,"typical":0.6,"tfs":1,"rep_pen":1.1,"rep_pen_range":1024,"rep_pen_slope":0.7,"sampler_order":[6,4,5,1,0,2,3]},{"preset":"TavernAI","description":"Preset used in TavernAI.","temp":0.79,"genamt":80,"top_k":0,"top_p":0.9,"min_p":0.0,"top_a":0,"typical":1,"tfs":0.95,"rep_pen":1.19,"rep_pen_range":1024,"rep_pen_slope":0.9,"sampler_order":[6,0,1,2,3,4,5]},{"preset":"Storywriter 6B","description":"Optimized settings for relevant output.","genamt":80,"rep_pen":1.1,"rep_pen_range":2048,"rep_pen_slope":0.2,"sampler_order":[6,5,0,2,3,1,4],"temp":0.72,"tfs":1,"top_a":0,"top_k":0,"top_p":0.73,"min_p":0.0,"typical":1},{"preset":"Coherent Creativity 6B","description":"A good balance between coherence, creativity, and quality of prose.","genamt":80,"rep_pen":1.2,"rep_pen_range":2048,"rep_pen_slope":0,"sampler_order":[6,5,0,2,3,1,4],"temp":0.51,"tfs":0.99,"top_a":0,"top_k":0,"top_p":1,"min_p":0.0,"typical":1},{"preset":"Luna Moth 6B","description":"A great degree of creativity without losing coherency.","temp":1.5,"genamt":80,"top_k":85,"top_p":0.24,"min_p":0.0,"top_a":0,"typical":1,"tfs":1,"rep_pen":1.1,"rep_pen_range":2048,"rep_pen_slope":0,"sampler_order":[6,5,0,2,3,1,4]},{"preset":"Best Guess 6B","description":"A subtle change with alternative context settings.","temp":0.8,"genamt":80,"top_k":100,"top_p":0.9,"min_p":0.0,"top_a":0,"typical":1,"tfs":1,"rep_pen":1.15,"rep_pen_range":2048,"rep_pen_slope":3.4,"sampler_order":[6,5,0,2,3,1,4]},{"preset":"Pleasing Results 6B","description":"Expectable output with alternative context settings.","temp":0.44,"genamt":80,"top_k":0,"top_p":1,"min_p":0.0,"top_a":0,"typical":1,"tfs":0.9,"rep_pen":1.15,"rep_pen_range":2048,"rep_pen_slope":6.8,"sampler_order":[6,5,0,2,3,1,4]},{"preset":"Genesis 13B","description":"Stable and logical, but with scattered creativity.","temp":0.63,"genamt":80,"top_k":0,"top_p":0.98,"min_p":0.0,"top_a":0,"typical":1,"tfs":0.98,"rep_pen":1.05,"rep_pen_range":2048,"rep_pen_slope":0.1,"sampler_order":[6,2,0,3,5,1,4]},{"preset":"Basic Coherence 13B","description":"Keep things on track.","temp":0.59,"genamt":80,"top_k":0,"top_p":1,"min_p":0.0,"top_a":0,"typical":1,"tfs":0.87,"rep_pen":1.1,"rep_pen_range":2048,"rep_pen_slope":0.3,"sampler_order":[6,5,0,2,3,1,4]},{"preset":"Ouroboros 13B","description":"Versatile, conforms well to poems, lists, chat, etc.","temp":1.07,"genamt":80,"top_k":100,"top_p":1,"min_p":0.0,"top_a":0,"typical":1,"tfs":0.93,"rep_pen":1.05,"rep_pen_range":404,"rep_pen_slope":0.8,"sampler_order":[6,0,5,3,2,1,4]},{"preset":"Ace of Spades 13B","description":"Expressive, while still staying focused.","temp":1.15,"genamt":80,"top_k":0,"top_p":0.95,"min_p":0.0,"top_a":0,"typical":1,"tfs":0.8,"rep_pen":1.05,"rep_pen_range":2048,"rep_pen_slope":7,"sampler_order":[6,3,2,0,5,1,4]},{"preset":"Low Rider 13B","description":"Reliable, aimed at story development.","temp":0.94,"genamt":80,"top_k":12,"top_p":1,"min_p":0.0,"top_a":0,"typical":1,"tfs":0.94,"rep_pen":1.05,"rep_pen_range":2048,"rep_pen_slope":0.2,"sampler_order":[6,5,0,2,3,1,4]},{"preset":"Pro Writer 13B","description":"Optimal setting for readability, based on AI-powered mass statistical analysis of Euterpe output.","temp":1.35,"genamt":80,"top_k":0,"top_p":1,"min_p":0.0,"top_a":0,"typical":1,"tfs":0.69,"rep_pen":1.15,"rep_pen_range":2048,"rep_pen_slope":0.1,"sampler_order":[6,3,2,5,0,1,4]},{"preset":"Default 20B","description":"Good starting settings for NeoX 20B.","temp":0.6,"genamt":80,"top_k":0,"top_p":0.9,"min_p":0.0,"top_a":0,"typical":1,"tfs":1,"rep_pen":1.04,"rep_pen_range":1024,"rep_pen_slope":0.7,"sampler_order":[6,0,1,2,3,4,5]}
];
function polyfills()
@ -6821,6 +6824,7 @@ Current version: 89
handle_bot_name_onchange();
document.getElementById("instruct_starttag").value = localsettings.instruct_starttag;
document.getElementById("instruct_endtag").value = localsettings.instruct_endtag;
document.getElementById("min_p").value = localsettings.min_p;
document.getElementById("top_k").value = localsettings.top_k;
document.getElementById("top_a").value = localsettings.top_a;
document.getElementById("typ_s").value = localsettings.typ_s;
@ -6931,6 +6935,7 @@ Current version: 89
temp_changingpreset = true;
document.getElementById("temperature").value = document.getElementById("temperature_slide").value = found.temp;
document.getElementById("max_length").value = document.getElementById("max_length_slide").value = found.genamt;
document.getElementById("min_p").value = found.min_p;
document.getElementById("top_k").value = found.top_k;
document.getElementById("top_p").value = document.getElementById("top_p_slide").value = found.top_p;
document.getElementById("top_a").value = found.top_a;
@ -7065,6 +7070,7 @@ Current version: 89
if (localsettings.instruct_endtag == null || localsettings.instruct_endtag == "") {
localsettings.instruct_endtag = "\\n### Response:\\n";
}
localsettings.min_p = document.getElementById("min_p").value;
localsettings.top_k = document.getElementById("top_k").value;
localsettings.top_a = document.getElementById("top_a").value;
localsettings.typ_s = document.getElementById("typ_s").value;
@ -7115,6 +7121,7 @@ Current version: 89
localsettings.rep_pen_range = cleannum(localsettings.rep_pen_range, 0, 8192);
localsettings.rep_pen_slope = cleannum(localsettings.rep_pen_slope, 0, 20);
localsettings.top_p = cleannum(localsettings.top_p, 0.002, 1);
localsettings.min_p = cleannum(localsettings.min_p, 0.0, 1);
localsettings.top_k = cleannum(Math.floor(localsettings.top_k), 0, 300);
localsettings.top_a = cleannum(localsettings.top_a, 0, 1);
localsettings.typ_s = cleannum(localsettings.typ_s, 0, 1);
@ -7376,6 +7383,9 @@ Current version: 89
submit_payload.params.mirostat_tau = localsettings.miro_tau;
submit_payload.params.mirostat_eta = localsettings.miro_eta;
}
//also supports min_p, in that it wont crash, so add it on. it will be ignored if not found
submit_payload.params.min_p = localsettings.min_p;
}
//v2 api specific fields
@ -8169,6 +8179,9 @@ Current version: 89
submit_payload.params.mirostat_tau = localsettings.miro_tau;
submit_payload.params.mirostat_eta = localsettings.miro_eta;
}
//also supports min_p, in that it wont crash, so add it on. it will be ignored if not found
submit_payload.params.min_p = localsettings.min_p;
}
if((custom_kobold_endpoint != "" && is_using_kcpp_with_grammar()))
@ -11654,6 +11667,16 @@ Current version: 89
<td><input class="" type="text" inputmode="decimal" placeholder="0" value="0"
id="tfs_s"></td>
</tr>
<tr>
<th title="Min-P Sampling. 0 to Deactivate.">Min-P</th>
</tr>
<tr>
<td><input class="" type="text" inputmode="decimal" placeholder="0" value="0"
id="min_p"></td>
</tr>
</table>
</div>

11
koboldcpp.py
View file

@ -55,6 +55,7 @@ class generation_inputs(ctypes.Structure):
("top_k", ctypes.c_int),
("top_a", ctypes.c_float),
("top_p", ctypes.c_float),
("min_p", ctypes.c_float),
("typical_p", ctypes.c_float),
("tfs", ctypes.c_float),
("rep_pen", ctypes.c_float),
@ -286,7 +287,7 @@ def load_model(model_filename):
ret = handle.load_model(inputs)
return ret
def generate(prompt,max_length=20, max_context_length=512, temperature=0.8, top_k=120, top_a=0.0, top_p=0.85, typical_p=1.0, tfs=1.0, rep_pen=1.1, rep_pen_range=128, mirostat=0, mirostat_tau=5.0, mirostat_eta=0.1, sampler_order=[6,0,1,3,4,2,5], seed=-1, stop_sequence=[], use_default_badwordsids=False, stream_sse=False, grammar='', grammar_retain_state=False, genkey=''):
def generate(prompt,max_length=32, max_context_length=512, temperature=0.7, top_k=100, top_a=0.0, top_p=0.92, min_p=0.0, typical_p=1.0, tfs=1.0, rep_pen=1.1, rep_pen_range=128, mirostat=0, mirostat_tau=5.0, mirostat_eta=0.1, sampler_order=[6,0,1,3,4,2,5], seed=-1, stop_sequence=[], use_default_badwordsids=False, stream_sse=False, grammar='', grammar_retain_state=False, genkey=''):
global maxctx, args, currentusergenkey, totalgens
inputs = generation_inputs()
outputs = ctypes.create_unicode_buffer(ctypes.sizeof(generation_outputs))
@ -303,6 +304,7 @@ def generate(prompt,max_length=20, max_context_length=512, temperature=0.8, top_
inputs.top_k = top_k
inputs.top_a = top_a
inputs.top_p = top_p
inputs.min_p = min_p
inputs.typical_p = typical_p
inputs.tfs = tfs
inputs.rep_pen = rep_pen
@ -463,10 +465,11 @@ class ServerRequestHandler(http.server.SimpleHTTPRequestHandler):
prompt=genparams.get('prompt', ""),
max_context_length=genparams.get('max_context_length', maxctx),
max_length=genparams.get('max_length', 80),
temperature=genparams.get('temperature', 0.8),
top_k=genparams.get('top_k', 120),
temperature=genparams.get('temperature', 0.7),
top_k=genparams.get('top_k', 100),
top_a=genparams.get('top_a', 0.0),
top_p=genparams.get('top_p', 0.85),
top_p=genparams.get('top_p', 0.92),
min_p=genparams.get('min_p', 0.0),
typical_p=genparams.get('typical', 1.0),
tfs=genparams.get('tfs', 1.0),
rep_pen=genparams.get('rep_pen', 1.1),