vbatts/llama.cpp
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

increase default blas batch size

Browse source
This commit is contained in:
Concedo 2023-04-09 15:27:43 +08:00
parent 4d1825263b
commit b91abc3316
3 changed files with 3 additions and 2 deletions
Download patch file Download diff file Expand all files Collapse all files

2
gpttype_adapter.cpp
View file

@ -209,7 +209,7 @@ generation_outputs gpttype_generate(const generation_inputs inputs, generation_o
int original_threads = params.n_threads; int original_threads = params.n_threads;
if (blasmode) if (blasmode)
{ {
params.n_batch = 512; params.n_batch = 1024;
params.n_threads = 1; params.n_threads = 1;
} }

1
koboldcpp.py
View file

@ -377,6 +377,7 @@ if __name__ == '__main__':
portgroup.add_argument("l_port", help="Port to listen on (deprecated)", default=defaultport, nargs="?", type=int, action='store') portgroup.add_argument("l_port", help="Port to listen on (deprecated)", default=defaultport, nargs="?", type=int, action='store')
parser.add_argument("--host", help="Host IP to listen on. If empty, all routable interfaces are accepted.", default="") parser.add_argument("--host", help="Host IP to listen on. If empty, all routable interfaces are accepted.", default="")
#os.environ["OMP_NUM_THREADS"] = '12'
# psutil.cpu_count(logical=False) # psutil.cpu_count(logical=False)
physical_core_limit = 1 physical_core_limit = 1
if os.cpu_count()!=None and os.cpu_count()>1: if os.cpu_count()!=None and os.cpu_count()>1:

2
llama_adapter.cpp
View file

@ -158,7 +158,7 @@ generation_outputs llama_generate(const generation_inputs inputs, generation_out
int original_threads = params.n_threads; int original_threads = params.n_threads;
if (blasmode) if (blasmode)
{ {
params.n_batch = 512; params.n_batch = 1024;
params.n_threads = 1; params.n_threads = 1;
} }