vbatts/llama.cpp
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

lite bugfixes, buffer size changes, fixed a topk bug.

Browse source
This commit is contained in:
Concedo 2023-06-06 22:38:25 +08:00
parent 2e5edc80e0
commit c046db5197
5 changed files with 12 additions and 8 deletions
Download patch file Download diff file Expand all files Collapse all files

8
klite.embd
View file

File diff suppressed because one or more lines are too long

6
koboldcpp.py
View file

@ -210,7 +210,7 @@ maxctx = 2048
maxlen = 256 maxlen = 256
modelbusy = False modelbusy = False
defaultport = 5001 defaultport = 5001
KcppVersion = "1.28" KcppVersion = "1.29"
class ServerRequestHandler(http.server.SimpleHTTPRequestHandler): class ServerRequestHandler(http.server.SimpleHTTPRequestHandler):
sys_version = "" sys_version = ""
@ -328,7 +328,7 @@ class ServerRequestHandler(http.server.SimpleHTTPRequestHandler):
max_context_length=genparams.get('max_context_length', maxctx), max_context_length=genparams.get('max_context_length', maxctx),
max_length=genparams.get('max_length', 50), max_length=genparams.get('max_length', 50),
temperature=genparams.get('temperature', 0.8), temperature=genparams.get('temperature', 0.8),
top_k=genparams.get('top_k', 120), top_k=int(genparams.get('top_k', 120)),
top_a=genparams.get('top_a', 0.0), top_a=genparams.get('top_a', 0.0),
top_p=genparams.get('top_p', 0.85), top_p=genparams.get('top_p', 0.85),
typical_p=genparams.get('typical', 1.0), typical_p=genparams.get('typical', 1.0),
@ -345,7 +345,7 @@ class ServerRequestHandler(http.server.SimpleHTTPRequestHandler):
prompt=newprompt, prompt=newprompt,
max_length=genparams.get('max', 50), max_length=genparams.get('max', 50),
temperature=genparams.get('temperature', 0.8), temperature=genparams.get('temperature', 0.8),
top_k=genparams.get('top_k', 120), top_k=int(genparams.get('top_k', 120)),
top_a=genparams.get('top_a', 0.0), top_a=genparams.get('top_a', 0.0),
top_p=genparams.get('top_p', 0.85), top_p=genparams.get('top_p', 0.85),
typical_p=genparams.get('typical', 1.0), typical_p=genparams.get('typical', 1.0),

2
otherarch/gpt2_v3.cpp
View file

@ -370,7 +370,7 @@ bool gpt2_eval(
static void * buf = malloc(buf_size); static void * buf = malloc(buf_size);
if (mem_per_token > 0 && (mem_per_token*N*2 + 64u*1024*1024) > buf_size) { if (mem_per_token > 0 && (mem_per_token*N*2 + 64u*1024*1024) > buf_size) {
const size_t buf_size_new = 320u*1024*1024 + 1.7*(mem_per_token*N); // add 10% to account for ggml object overhead const size_t buf_size_new = 320u*1024*1024 + 1.6*(mem_per_token*N); // add 10% to account for ggml object overhead
//printf("\n%s: reallocating buffer from %zu to %zu bytes\n", __func__, buf_size, buf_size_new); //printf("\n%s: reallocating buffer from %zu to %zu bytes\n", __func__, buf_size, buf_size_new);
// reallocate // reallocate

2
otherarch/gptj_v3.cpp
View file

@ -368,7 +368,7 @@ bool gptj_eval(
static void * buf = malloc(buf_size); static void * buf = malloc(buf_size);
if (mem_per_token > 0 && (mem_per_token*N*2 + 64u*1024*1024) > buf_size) { if (mem_per_token > 0 && (mem_per_token*N*2 + 64u*1024*1024) > buf_size) {
const size_t buf_size_new = 320u*1024*1024 + 1.7*(mem_per_token*N); // add 10% to account for ggml object overhead const size_t buf_size_new = 320u*1024*1024 + 1.6*(mem_per_token*N); // add 10% to account for ggml object overhead
//printf("\n%s: reallocating buffer from %zu to %zu bytes\n", __func__, buf_size, buf_size_new); //printf("\n%s: reallocating buffer from %zu to %zu bytes\n", __func__, buf_size, buf_size_new);
// reallocate // reallocate

2
otherarch/neox_v3.cpp
View file

@ -410,7 +410,7 @@ bool gpt_neox_eval(
static void * buf = malloc(buf_size); static void * buf = malloc(buf_size);
if (mem_per_token > 0 && (mem_per_token*N*2 + 64u*1024*1024) > buf_size) { if (mem_per_token > 0 && (mem_per_token*N*2 + 64u*1024*1024) > buf_size) {
const size_t buf_size_new = 360u*1024*1024 + 1.7*(mem_per_token*N); // add 10% to account for ggml object overhead const size_t buf_size_new = 360u*1024*1024 + 1.6*(mem_per_token*N); // add 10% to account for ggml object overhead
//printf("\n%s: reallocating buffer from %zu to %zu bytes\n", __func__, buf_size, buf_size_new); //printf("\n%s: reallocating buffer from %zu to %zu bytes\n", __func__, buf_size, buf_size_new);
// reallocate // reallocate